diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java new file mode 100644 index 000000000000..329a4ecd75be --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java @@ -0,0 +1,627 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.InternalServerErrorException; + +import org.apache.catalina.connector.ClientAbortException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.tomcat.util.http.FastHttpDateFormat; +import org.dspace.app.rest.utils.ContextUtil; +import org.dspace.app.rest.utils.HttpHeadersInitializer; +import org.dspace.app.statistics.clarin.ClarinMatomoBitstreamTracker; +import org.dspace.authorize.AuthorizeException; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.content.Bitstream; +import org.dspace.content.BitstreamFormat; +import org.dspace.content.Item; +import org.dspace.core.Constants; +import org.dspace.core.Context; +import org.dspace.eperson.EPerson; +import org.dspace.services.ConfigurationService; +import org.dspace.services.EventService; +import org.dspace.storage.bitstore.S3BitStoreService; +import org.dspace.storage.bitstore.service.S3DirectDownloadService; +import org.dspace.usage.UsageEvent; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.util.UriUtils; + +/** + * HuggingFace-Hub-compatible facade controller (read-only). + * + *

Exposes a subset of the {@code huggingface_hub} REST API shape (model info, tree listing, file + * download/"resolve" and {@code whoami-v2}) backed by {@link ClarinHuggingFaceService}, so that DSpace + * Items flagged as "machine learning models" (see the {@code hf.api.*} configuration keys) can be + * consumed directly by HuggingFace-Hub-aware tooling (e.g. the {@code huggingface_hub} Python client) + * using the Item handle as the repository id.

+ * + *

The file-download ({@code resolve}) route streams (or, when configured for S3 direct download, + * redirects to) the ORIGINAL bitstreams of an exposed Item.

+ * + *

Note: {@code @PreAuthorize} is not used because authorization depends on the resolved Item (looked + * up by handle), not on a UUID path variable. Authorization is explicitly checked via + * {@link AuthorizeService#authorizeAction} after the Item is resolved, mirroring + * {@code BitstreamByHandleRestController}.

+ * + * @author DSpace at UFAL + */ +@RestController +@RequestMapping("/api/hf") +public class ClarinHuggingFaceController { + + private static final Logger log = LogManager.getLogger(ClarinHuggingFaceController.class); + + private static final String ERROR_CODE_HEADER = "X-Error-Code"; + + private static final String REPO_COMMIT_HEADER = "X-Repo-Commit"; + + private static final String LINKED_ETAG_HEADER = "X-Linked-Etag"; + + private static final String LINKED_SIZE_HEADER = "X-Linked-Size"; + + private static final String RESOLVE_MARKER = "/resolve/"; + + // Most file systems are configured to use block sizes of 4096 or 8192 and our buffer should be a + // multiple of that (mirrors BitstreamRestController / BitstreamByHandleRestController). + private static final int BUFFER_SIZE = 4096 * 10; + + @Autowired + private ClarinHuggingFaceService huggingFaceService; + + @Autowired + private AuthorizeService authorizeService; + + @Autowired + private ConfigurationService configurationService; + + @Autowired + private EventService eventService; + + @Autowired + private ClarinMatomoBitstreamTracker matomoBitstreamTracker; + + @Autowired + private S3DirectDownloadService s3DirectDownloadService; + + @Autowired + private S3BitStoreService s3BitStoreService; + + /** + * Get the model info of an exposed Item, at the implicit "main" revision. + * + * @param prefix the handle prefix (e.g. "11234") + * @param suffix the handle suffix (e.g. "1-5814") + * @param request the HTTP request + * @return the model info, or an error response + * @throws SQLException if a database error occurs + */ + @GetMapping("/api/models/{prefix}/{suffix}") + public ResponseEntity modelInfo(@PathVariable String prefix, @PathVariable String suffix, + HttpServletRequest request) throws SQLException { + if (!huggingFaceService.isEnabled()) { + return ResponseEntity.notFound().build(); + } + + Context context = ContextUtil.obtainContext(request); + try { + ResolvedRepository repository = resolveAuthorizedRepository(context, prefix, suffix); + HuggingFaceModelInfo modelInfo = + huggingFaceService.toModelInfo(context, repository.item, repository.sha); + context.complete(); + return ResponseEntity.ok(modelInfo); + } catch (HuggingFaceApiException e) { + return e.toResponse(); + } + } + + /** + * Get the model info of an exposed Item, at a specific (validated) revision. + * + * @param prefix the handle prefix (e.g. "11234") + * @param suffix the handle suffix (e.g. "1-5814") + * @param revision the requested revision; must be {@code "main"} or the item's current sha + * @param request the HTTP request + * @return the model info, or an error response + * @throws SQLException if a database error occurs + */ + @GetMapping("/api/models/{prefix}/{suffix}/revision/{revision}") + public ResponseEntity modelInfoAtRevision(@PathVariable String prefix, @PathVariable String suffix, + @PathVariable String revision, HttpServletRequest request) throws SQLException { + if (!huggingFaceService.isEnabled()) { + return ResponseEntity.notFound().build(); + } + + Context context = ContextUtil.obtainContext(request); + try { + ResolvedRepository repository = resolveAuthorizedRepository(context, prefix, suffix); + requireValidRevision(revision, repository.sha); + HuggingFaceModelInfo modelInfo = + huggingFaceService.toModelInfo(context, repository.item, repository.sha); + context.complete(); + return ResponseEntity.ok(modelInfo); + } catch (HuggingFaceApiException e) { + return e.toResponse(); + } + } + + /** + * Get the flat file tree of an exposed Item, at a specific (validated) revision. + * + * @param prefix the handle prefix (e.g. "11234") + * @param suffix the handle suffix (e.g. "1-5814") + * @param revision the requested revision; must be {@code "main"} or the item's current sha + * @param recursive accepted and ignored (single-page flat listing regardless of its value) + * @param request the HTTP request + * @return the flat list of tree entries, or an error response + * @throws SQLException if a database error occurs + */ + @GetMapping("/api/models/{prefix}/{suffix}/tree/{revision}") + public ResponseEntity tree(@PathVariable String prefix, @PathVariable String suffix, + @PathVariable String revision, + @RequestParam(required = false) String recursive, + HttpServletRequest request) throws SQLException { + if (!huggingFaceService.isEnabled()) { + return ResponseEntity.notFound().build(); + } + + Context context = ContextUtil.obtainContext(request); + try { + ResolvedRepository repository = resolveAuthorizedRepository(context, prefix, suffix); + requireValidRevision(revision, repository.sha); + List entries = huggingFaceService.toTreeEntries(context, repository.item); + context.complete(); + return ResponseEntity.ok(entries); + } catch (HuggingFaceApiException e) { + return e.toResponse(); + } + } + + /** + * List exposed Items ("models"), HuggingFace-Hub {@code GET /api/models} style. + * + *

This is a metadata-only listing of exposed items: only items that pass + * {@link ClarinHuggingFaceService#isExposed(Context, Item)} (fail-closed, public-by-configuration) AND + * that the current context user is authorized to {@code READ} are ever returned -- the {@code READ} + * check is performed by {@link ClarinHuggingFaceService#findExposedItems(Context, String, int)} itself, + * before pagination, so that embargoed/private items never count towards {@code limit} and are never + * leaked to a user who could not otherwise read them.

+ * + * @param search an optional, case-insensitive substring to match against the item title + * @param limit the maximum number of items to return (default 20, capped at 100) + * @param request the HTTP request + * @return the matching model-info objects, as a JSON array + * @throws SQLException if a database error occurs + */ + @GetMapping("/api/models") + public ResponseEntity listModels(@RequestParam(required = false) String search, + @RequestParam(required = false, defaultValue = "20") int limit, + HttpServletRequest request) throws SQLException { + if (!huggingFaceService.isEnabled()) { + return ResponseEntity.notFound().build(); + } + + Context context = ContextUtil.obtainContext(request); + List models = new ArrayList<>(); + for (Item item : huggingFaceService.findExposedItems(context, search, limit)) { + String sha = huggingFaceService.computeShaForItem(context, item); + models.add(huggingFaceService.toModelInfo(context, item, sha)); + } + context.complete(); + return ResponseEntity.ok(models); + } + + /** + * Placeholder for the HuggingFace Hub {@code whoami-v2} endpoint. Always responds with 401, since + * this facade does not (yet) support Bearer-token authentication. + * + * @return a 401 response with a small JSON error body + */ + @GetMapping("/api/whoami-v2") + public ResponseEntity whoAmI() { + // TODO: phase 2 - map Authorization: Bearer to CLARIN Personal Access Token here + return ResponseEntity.status(HttpStatus.UNAUTHORIZED) + .body(Collections.singletonMap("error", "Invalid user token")); + } + + /** + * Download (GET) or probe (HEAD) a single ORIGINAL bitstream of an exposed Item, HuggingFace-Hub + * "resolve" style: {@code /{prefix}/{suffix}/resolve/{revision}/{filename}}. + * + *

The filename is everything after the revision path segment (it may contain slashes and + * URL-encoded characters, decoded once). HEAD requests are always answered locally with the metadata + * headers ({@code X-Repo-Commit}, {@code ETag}, {@code X-Linked-Etag}, {@code Content-Length}, + * {@code X-Linked-Size}, {@code Accept-Ranges}) and an empty body; they are never redirected. GET + * requests are either redirected (302) to an S3 presigned URL when direct S3 download is enabled, or + * streamed with full Range/conditional-request support.

+ * + * @param prefix the handle prefix (e.g. "11234") + * @param suffix the handle suffix (e.g. "1-5814") + * @param revision the requested revision; must be {@code "main"} or the item's current sha + * @param request the HTTP request + * @param response the HTTP response + * @return the response entity (file content, redirect or error), or {@code null} when the response + * has already been committed (e.g. 304/412 shortcuts or a client abort) + * @throws SQLException if a database error occurs + * @throws IOException if an I/O error occurs while initialising the response + */ + @RequestMapping(method = {RequestMethod.GET, RequestMethod.HEAD}, + value = "/{prefix}/{suffix}/resolve/{revision}/**") + public ResponseEntity resolveFile(@PathVariable String prefix, @PathVariable String suffix, + @PathVariable String revision, HttpServletRequest request, HttpServletResponse response) + throws SQLException, IOException { + if (!huggingFaceService.isEnabled()) { + return ResponseEntity.notFound().build(); + } + + Context context = ContextUtil.obtainContext(request); + try { + if (Objects.isNull(context)) { + log.error("Cannot obtain the context from the request."); + throw new HuggingFaceApiException(HttpStatus.INTERNAL_SERVER_ERROR, null, "Internal error."); + } + + String handle = prefix + "/" + suffix; + Item item = huggingFaceService.resolveExposedItem(context, handle); + if (item == null) { + throw new HuggingFaceApiException(HttpStatus.NOT_FOUND, "RepoNotFound", "Repository not found"); + } + + String sha = huggingFaceService.computeShaForItem(context, item); + requireValidRevision(revision, sha); + + String filename = extractResolveFilename(request); + if (StringUtils.isBlank(filename)) { + throw new HuggingFaceApiException(HttpStatus.NOT_FOUND, "EntryNotFound", "Entry not found"); + } + + Bitstream bitstream = huggingFaceService.getOriginalFiles(item).get(filename); + if (bitstream == null) { + throw new HuggingFaceApiException(HttpStatus.NOT_FOUND, "EntryNotFound", "Entry not found"); + } + + try { + // Authorization is checked explicitly here (not via @PreAuthorize) because the bitstream + // identity is resolved from handle + filename, not from a UUID path variable. + authorizeService.authorizeAction(context, bitstream, Constants.READ); + } catch (AuthorizeException e) { + // TODO: phase 2 - Bearer/PAT auth: map Authorization: Bearer to a CLARIN Personal Access + // Token and re-check authorization with the resolved EPerson before falling back to the + // anonymous 401/403 distinction below. + log.warn("Unauthorized access to file '{}' of HuggingFace repository '{}'.", filename, handle); + HttpStatus status = context.getCurrentUser() == null + ? HttpStatus.UNAUTHORIZED : HttpStatus.FORBIDDEN; + String landingPage = configurationService.getProperty("dspace.ui.url") + "/handle/" + handle; + throw new HuggingFaceApiException(status, null, + "You are not authorized to download this file. Please visit the item landing page at " + + landingPage + " to request access."); + } + + return serveBitstream(context, request, response, item, bitstream, sha); + } catch (HuggingFaceApiException e) { + return e.toResponse(); + } + } + + /** + * Serve a resolved and authorized bitstream: HEAD locally, GET via an S3 presigned-URL redirect when + * direct S3 download is enabled, or by streaming the content otherwise. + * + * @param context the DSpace context + * @param request the HTTP request + * @param response the HTTP response + * @param item the owning item (source of {@code Last-Modified}) + * @param bitstream the bitstream to serve + * @param sha the item's current sha (sent as {@code X-Repo-Commit}) + * @return the response entity, or {@code null} when the response has already been committed + * @throws SQLException if a database error occurs + * @throws IOException if an I/O error occurs while initialising the response + */ + private ResponseEntity serveBitstream(Context context, HttpServletRequest request, + HttpServletResponse response, Item item, Bitstream bitstream, String sha) + throws SQLException, IOException { + BitstreamFormat format = bitstream.getFormat(context); + String mimetype = format != null ? format.getMIMEType() : MediaType.APPLICATION_OCTET_STREAM_VALUE; + String name = StringUtils.isNotBlank(bitstream.getName()) + ? bitstream.getName() : bitstream.getID().toString(); + String quotedEtag = "\"" + bitstream.getChecksum() + "\""; + long sizeBytes = bitstream.getSizeBytes(); + long lastModifiedMillis = item.getLastModified() != null ? item.getLastModified().getTime() : 0L; + + if (RequestMethod.HEAD.name().equals(request.getMethod())) { + // HEAD is always answered locally (never redirected to S3), so that HuggingFace clients can + // read the metadata headers without following a presigned URL. + HttpHeaders headers = buildCommonHeaders(sha, quotedEtag, sizeBytes, mimetype, lastModifiedMillis); + context.complete(); + return ResponseEntity.ok().headers(headers).build(); + } + + // We only log a download for requests without a Range header, because a client always sends a + // regular request first to check for Range support (mirrors BitstreamByHandleRestController). + if (StringUtils.isBlank(request.getHeader("Range"))) { + eventService.fireEvent( + new UsageEvent( + UsageEvent.Action.VIEW, + request, + context, + bitstream)); + + // Track the download in Matomo - only if the downloading has started (the condition is + // inside the method) + matomoBitstreamTracker.trackBitstreamDownload(context, request, bitstream, false); + } + + try { + boolean s3DirectDownload = configurationService.getBooleanProperty("s3.download.direct.enabled"); + boolean s3AssetstoreEnabled = configurationService.getBooleanProperty("assetstore.s3.enabled"); + if (s3DirectDownload && s3AssetstoreEnabled) { + // Download only files which are stored in the `ORIGINAL` bundle, because some specific + // files are not correctly downloaded and displayed in the UI when using presigned URLs. + boolean hasOriginalBundle = bitstream.getBundles().stream() + .anyMatch(bundle -> Constants.CONTENT_BUNDLE_NAME.equals(bundle.getName())); + if (hasOriginalBundle) { + HttpHeaders headers = + buildCommonHeaders(sha, quotedEtag, sizeBytes, mimetype, lastModifiedMillis); + // Close the DB connection before redirecting + context.complete(); + return redirectToS3DownloadUrl(headers, name, bitstream.getInternalId()); + } + } + + EPerson currentUser = context.getCurrentUser(); + org.dspace.app.rest.utils.BitstreamResource bitstreamResource = + new org.dspace.app.rest.utils.BitstreamResource(name, bitstream.getID(), + currentUser != null ? currentUser.getID() : null, + context.getSpecialGroupUuids(), false); + + HttpHeadersInitializer httpHeadersInitializer = new HttpHeadersInitializer() + .withBufferSize(BUFFER_SIZE) + .withFileName(name) + .withChecksum(quotedEtag) + .withLength(bitstreamResource.contentLength()) + .withMimetype(mimetype) + .withLastModified(lastModifiedMillis) + .withDisposition(HttpHeadersInitializer.CONTENT_DISPOSITION_ATTACHMENT) + .with(request) + .with(response); + + // We have all the data we need, close the connection to the database so that it doesn't + // stay open during download/streaming + context.complete(); + + if (httpHeadersInitializer.isValid()) { + HttpHeaders httpHeaders = httpHeadersInitializer.initialiseHeaders(); + httpHeaders.set(REPO_COMMIT_HEADER, sha); + httpHeaders.set(LINKED_ETAG_HEADER, quotedEtag); + httpHeaders.set(LINKED_SIZE_HEADER, String.valueOf(sizeBytes)); + return ResponseEntity.ok().headers(httpHeaders).body(bitstreamResource); + } + } catch (ClientAbortException ex) { + log.debug("Client aborted the request before the download was completed. " + + "Client is probably switching to a Range request.", ex); + } + return null; + } + + /** + * Build the metadata headers shared by every successful {@code resolve} response (HEAD, 302 redirect + * and streamed GET). + * + * @param sha the item's current sha + * @param quotedEtag the bitstream's MD5 checksum, wrapped in double quotes + * @param sizeBytes the bitstream size in bytes + * @param mimetype the bitstream MIME type + * @param lastModifiedMillis the item's last-modified timestamp, in epoch milliseconds + * @return the headers + */ + private HttpHeaders buildCommonHeaders(String sha, String quotedEtag, long sizeBytes, String mimetype, + long lastModifiedMillis) { + HttpHeaders headers = new HttpHeaders(); + headers.set(REPO_COMMIT_HEADER, sha); + headers.set(HttpHeaders.ETAG, quotedEtag); + headers.set(LINKED_ETAG_HEADER, quotedEtag); + headers.set(HttpHeaders.CONTENT_LENGTH, String.valueOf(sizeBytes)); + headers.set(LINKED_SIZE_HEADER, String.valueOf(sizeBytes)); + headers.set(HttpHeaders.ACCEPT_RANGES, "bytes"); + headers.set(HttpHeaders.CONTENT_TYPE, mimetype); + headers.set(HttpHeaders.LAST_MODIFIED, FastHttpDateFormat.formatDate(lastModifiedMillis)); + return headers; + } + + /** + * Extract the requested filename from the raw request URI of a {@code resolve} request: everything + * after the {@code /resolve/} marker and the following (revision) path segment, decoded once. This + * supports filenames containing slashes, spaces, parentheses and other URL-encoded characters. + * + * @param request the HTTP request + * @return the decoded filename, or {@code null} if the URI contains no filename part + */ + private String extractResolveFilename(HttpServletRequest request) { + String uri = request.getRequestURI(); + int markerIndex = uri.indexOf(RESOLVE_MARKER); + if (markerIndex < 0) { + return null; + } + String rest = uri.substring(markerIndex + RESOLVE_MARKER.length()); + int slashIndex = rest.indexOf('/'); + if (slashIndex < 0) { + return null; + } + String rawFilename = rest.substring(slashIndex + 1); + return UriUtils.decode(rawFilename, StandardCharsets.UTF_8); + } + + /** + * This method will handle the S3 direct download by generating a presigned URL for the bitstream and + * returning a redirect response to the client (mirrors {@code BitstreamRestController}). + * + * @param httpHeaders headers needed to form a proper response when returning the Bitstream/File + * @param bitName name of the bitstream + * @param bitInternalId internal id of the bitstream + * @return ResponseEntity with the location header set to the presigned URL + */ + private ResponseEntity redirectToS3DownloadUrl(HttpHeaders httpHeaders, String bitName, + String bitInternalId) { + try { + String bucket = configurationService.getProperty("assetstore.s3.bucketName", ""); + if (StringUtils.isBlank(bucket)) { + throw new InternalServerErrorException("S3 bucket name is not configured"); + } + + // Get the full path to the bitstream in the S3 bucket + String bitstreamPath = s3BitStoreService.getFullKey(bitInternalId); + if (StringUtils.isBlank(bitstreamPath)) { + throw new InternalServerErrorException("Failed to get bitstream path for internal ID: " + + bitInternalId); + } + + // Generate a presigned URL for the bitstream with a configurable expiration time + int expirationTime = configurationService.getIntProperty("s3.download.direct.expiration", 3600); + log.debug("Generating presigned URL with expiration time of {} seconds", expirationTime); + String presignedUrl = + s3DirectDownloadService.generatePresignedUrl(bucket, bitstreamPath, expirationTime, bitName); + + if (StringUtils.isBlank(presignedUrl)) { + throw new InternalServerErrorException("Failed to generate presigned URL for bitstream: " + + bitInternalId); + } + + // Set the Location header to the presigned URL - this will redirect the client to the S3 URL + httpHeaders.setLocation(URI.create(presignedUrl)); + return ResponseEntity.status(HttpStatus.FOUND).headers(httpHeaders).build(); + } catch (Exception e) { + throw new InternalServerErrorException("Error generating S3 presigned URL for bitstream: " + + bitInternalId, e); + } + } + + /** + * Common flow shared by the model-info, revision and tree endpoints: resolve the handle to an + * exposed Item, check READ authorization on it, and compute its current sha. + * + * @param context the DSpace context + * @param prefix the handle prefix + * @param suffix the handle suffix + * @return the resolved item together with its current sha + * @throws SQLException if a database error occurs while resolving the handle + * @throws HuggingFaceApiException if the handle does not resolve to an exposed item, or the + * current user is not authorized to read it + */ + private ResolvedRepository resolveAuthorizedRepository(Context context, String prefix, String suffix) + throws SQLException { + String handle = prefix + "/" + suffix; + + if (Objects.isNull(context)) { + log.error("Cannot obtain the context from the request."); + throw new HuggingFaceApiException(HttpStatus.INTERNAL_SERVER_ERROR, null, "Internal error."); + } + + Item item = huggingFaceService.resolveExposedItem(context, handle); + if (item == null) { + throw new HuggingFaceApiException(HttpStatus.NOT_FOUND, "RepoNotFound", "Repository not found"); + } + + try { + authorizeService.authorizeAction(context, item, Constants.READ); + } catch (AuthorizeException e) { + // TODO: phase 2 - Bearer token auth: map Authorization: Bearer to a CLARIN Personal Access + // Token and re-check authorization with the resolved EPerson before falling back to the + // anonymous 401/403 distinction below. + log.warn("Unauthorized access to HuggingFace repository '{}'.", handle); + HttpStatus status = context.getCurrentUser() == null ? HttpStatus.UNAUTHORIZED : HttpStatus.FORBIDDEN; + throw new HuggingFaceApiException(status, null, "You are not authorized to access this repository."); + } + + String sha = huggingFaceService.computeShaForItem(context, item); + return new ResolvedRepository(item, sha); + } + + /** + * Validate a requested revision against the resolved item's current sha. + * + * @param revision the requested revision + * @param currentSha the item's current sha + * @throws HuggingFaceApiException if the revision is not valid + */ + private void requireValidRevision(String revision, String currentSha) { + if (!ClarinHuggingFaceService.isValidRevision(revision, currentSha)) { + throw new HuggingFaceApiException(HttpStatus.NOT_FOUND, "RevisionNotFound", "Revision not found"); + } + } + + /** + * Simple holder for the result of {@link #resolveAuthorizedRepository(Context, String, String)}. + */ + private static final class ResolvedRepository { + + private final Item item; + + private final String sha; + + private ResolvedRepository(Item item, String sha) { + this.item = item; + this.sha = sha; + } + } + + /** + * Internal unchecked exception used to short-circuit the common resolve/authorize/validate-revision + * flow shared by the model-info, revision and tree endpoints, carrying the HTTP status, optional + * {@code X-Error-Code} header value and JSON error message to send back to the client. + */ + private static final class HuggingFaceApiException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final HttpStatus status; + + private final String errorCode; + + private final String message; + + private HuggingFaceApiException(HttpStatus status, String errorCode, String message) { + this.status = status; + this.errorCode = errorCode; + this.message = message; + } + + /** + * Build the JSON error {@link ResponseEntity} for this exception. + * + * @return the response entity + */ + private ResponseEntity toResponse() { + ResponseEntity.BodyBuilder builder = ResponseEntity.status(status); + if (errorCode != null) { + builder = builder.header(ERROR_CODE_HEADER, errorCode); + } + return builder.body(Collections.singletonMap("error", message)); + } + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java new file mode 100644 index 000000000000..bdf34f09a91e --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java @@ -0,0 +1,510 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.authorize.AuthorizeException; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.content.Bitstream; +import org.dspace.content.Bundle; +import org.dspace.content.Collection; +import org.dspace.content.DSpaceObject; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.service.ItemService; +import org.dspace.core.Constants; +import org.dspace.core.Context; +import org.dspace.handle.service.HandleService; +import org.dspace.services.ConfigurationService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Service backing the (future, Milestone 2) HuggingFace-Hub-compatible facade controller. + * + *

This service is responsible for:

+ *
    + *
  • deciding whether the facade is enabled at all ({@link #isEnabled()});
    • + *
  • deciding whether a given Item is allowed to be exposed through the facade + * ({@link #isExposed(Context, Item)});
    • + *
  • resolving a handle to an exposed Item ({@link #resolveExposedItem(Context, String)});
    • + *
  • computing a deterministic "sha" (revision) for an Item's ORIGINAL bitstreams + * ({@link #computeSha(String, long, Map)} / {@link #computeShaForItem(Context, Item)});
    • + *
  • validating a requested revision against the current sha ({@link #isValidRevision(String, String)});
    • + *
  • building the HuggingFace-Hub-shaped JSON model ({@link #toModelInfo(Context, Item, String)}) and + * file tree ({@link #toTreeEntries(Context, Item)}) for an Item.
    • + *
+ * + * @author DSpace at UFAL + */ +@Component +public class ClarinHuggingFaceService { + + private static final Logger log = LogManager.getLogger(ClarinHuggingFaceService.class); + + private static final String PROP_ENABLED = "hf.api.enabled"; + + private static final String PROP_FILTER_METADATA_FIELD = "hf.api.filter.metadata-field"; + + private static final String PROP_FILTER_VALUE = "hf.api.filter.value"; + + private static final String PROP_EXPOSED_COLLECTIONS = "hf.api.exposed.collections"; + + private static final String DEFAULT_FILTER_METADATA_FIELD = "dc.type"; + + private static final String MAIN_REVISION = "main"; + + private static final int DEFAULT_LIST_LIMIT = 20; + + private static final int MAX_LIST_LIMIT = 100; + + /** + * Guard so that the "no exposure filter configured" warning is only logged once per JVM lifetime, + * instead of once per checked Item. + */ + private static final AtomicBoolean NO_FILTER_CONFIGURED_WARNING_LOGGED = new AtomicBoolean(false); + + @Autowired + private ConfigurationService configurationService; + + @Autowired + private ItemService itemService; + + @Autowired + private HandleService handleService; + + @Autowired + private AuthorizeService authorizeService; + + /** + * Master switch for the HuggingFace-Hub-compatible facade. + * + * @return {@code true} if the facade is enabled via {@code hf.api.enabled}, {@code false} otherwise + */ + public boolean isEnabled() { + return configurationService.getBooleanProperty(PROP_ENABLED, false); + } + + /** + * Fail-closed exposure filter deciding whether a given Item may be exposed through the HuggingFace facade. + * + *

An Item is exposed iff:

+ *
    + *
  • the item is non-null, archived and not withdrawn, AND
    • + *
  • either (a) the configured metadata field ({@code hf.api.filter.metadata-field}) has a value on + * the item equal to the configured filter value ({@code hf.api.filter.value}) -- only evaluated when + * both properties are non-blank -- or (b) the item's owning collection handle is listed in + * {@code hf.api.exposed.collections}.
    • + *
+ * + *

If neither the metadata filter nor the collection allow-list is configured, a warning is logged + * (once) and the method fails closed, returning {@code false}.

+ * + * @param context the DSpace context + * @param item the item to check + * @return {@code true} if the item may be exposed through the HuggingFace facade + */ + public boolean isExposed(Context context, Item item) { + if (item == null || item.isWithdrawn() || !item.isArchived()) { + return false; + } + + String filterMetadataField = configurationService.getProperty(PROP_FILTER_METADATA_FIELD, + DEFAULT_FILTER_METADATA_FIELD); + String filterValue = configurationService.getProperty(PROP_FILTER_VALUE); + boolean filterConfigured = StringUtils.isNotBlank(filterMetadataField) && StringUtils.isNotBlank(filterValue); + + String[] exposedCollections = configurationService.getArrayProperty(PROP_EXPOSED_COLLECTIONS); + boolean collectionsConfigured = exposedCollections != null && exposedCollections.length > 0; + + if (!filterConfigured && !collectionsConfigured) { + if (NO_FILTER_CONFIGURED_WARNING_LOGGED.compareAndSet(false, true)) { + log.warn("Neither '{}'/'{}' nor '{}' is configured; the HuggingFace facade will not expose " + + "any item.", PROP_FILTER_METADATA_FIELD, PROP_FILTER_VALUE, + PROP_EXPOSED_COLLECTIONS); + } + return false; + } + + if (filterConfigured) { + List values = itemService.getMetadataByMetadataString(item, filterMetadataField); + for (MetadataValue value : values) { + if (StringUtils.equals(filterValue, value.getValue())) { + return true; + } + } + } + + if (collectionsConfigured) { + Collection owningCollection = item.getOwningCollection(); + String ownerHandle = owningCollection != null ? owningCollection.getHandle() : null; + if (ownerHandle != null) { + for (String exposedHandle : exposedCollections) { + if (StringUtils.equals(ownerHandle, exposedHandle)) { + return true; + } + } + } + } + + return false; + } + + /** + * Resolve a handle to an Item that is allowed to be exposed through the HuggingFace facade. + * + * @param context the DSpace context + * @param handle the handle to resolve (e.g. {@code "11234/1-5814"}) + * @return the exposed Item, or {@code null} if the handle does not resolve to an Item, or the resolved + * Item is not exposed + * @throws SQLException if a database error occurs while resolving the handle + */ + public Item resolveExposedItem(Context context, String handle) throws SQLException { + DSpaceObject dso = handleService.resolveToObject(context, handle); + if (!(dso instanceof Item)) { + return null; + } + Item item = (Item) dso; + return isExposed(context, item) ? item : null; + } + + /** + * Find items exposed through the HuggingFace facade, optionally restricted to items whose title + * contains {@code search} (case-insensitively), and capped at {@code limit} results. + * + *

Candidates are collected -- deliberately without any Solr dependence -- from up to two sources, + * which are merged and deduplicated by item UUID:

+ *
    + *
  • every item of every collection listed in {@code hf.api.exposed.collections}, resolved via + * {@link HandleService} and iterated with {@link ItemService#findByCollection(Context, Collection)}; + * and
    • + *
  • the items whose configured filter metadata field ({@code hf.api.filter.metadata-field}) has + * the configured filter value ({@code hf.api.filter.value}), via + * {@link ItemService#findArchivedByMetadataField(Context, String, String, String, String)}.
    • + *
+ * + *

Every candidate is (re-)checked against {@link #isExposed(Context, Item)}, fail-closed, which also + * filters out withdrawn/non-archived items that may still be reachable through the collection path. Each + * exposed candidate is additionally checked for {@code READ} authorization for the current context user + * ({@link AuthorizeService#authorizeActionBoolean(Context, DSpaceObject, int)}) before it counts towards + * {@code limit}, so that embargoed/private items are filtered out before pagination rather than after, + * and are never returned to a user who could not otherwise read them.

+ * + * @param context the DSpace context + * @param search an optional, case-insensitive substring to match against the item title (or name, if + * the item has no title); {@code null} or blank to not filter by title + * @param limit the maximum number of items to return; a non-positive value defaults to 20, and any + * value is capped at 100 + * @return the list of exposed, READ-authorized items matching {@code search}, capped at the effective + * limit; empty if neither exposure mechanism is configured, or no item is exposed, readable and + * matches + * @throws SQLException if a database error occurs + */ + public List findExposedItems(Context context, String search, int limit) throws SQLException { + int effectiveLimit = limit <= 0 ? DEFAULT_LIST_LIMIT : Math.min(limit, MAX_LIST_LIMIT); + + LinkedHashMap candidates = new LinkedHashMap<>(); + + String[] exposedCollections = configurationService.getArrayProperty(PROP_EXPOSED_COLLECTIONS); + if (exposedCollections != null) { + for (String handle : exposedCollections) { + DSpaceObject dso = handleService.resolveToObject(context, handle); + if (dso instanceof Collection) { + Iterator items = itemService.findByCollection(context, (Collection) dso); + while (items.hasNext()) { + Item item = items.next(); + candidates.putIfAbsent(item.getID(), item); + } + } + } + } + + String filterMetadataField = configurationService.getProperty(PROP_FILTER_METADATA_FIELD, + DEFAULT_FILTER_METADATA_FIELD); + String filterValue = configurationService.getProperty(PROP_FILTER_VALUE); + if (StringUtils.isNotBlank(filterMetadataField) && StringUtils.isNotBlank(filterValue)) { + String[] fieldParts = filterMetadataField.split("\\."); + String schema = fieldParts.length > 0 ? fieldParts[0] : null; + String element = fieldParts.length > 1 ? fieldParts[1] : null; + String qualifier = fieldParts.length > 2 ? fieldParts[2] : null; + try { + Iterator items = + itemService.findArchivedByMetadataField(context, schema, element, qualifier, filterValue); + while (items.hasNext()) { + Item item = items.next(); + candidates.putIfAbsent(item.getID(), item); + } + } catch (AuthorizeException e) { + log.warn("Unexpected AuthorizeException while listing HuggingFace-exposed items by metadata " + + "filter '{}' = '{}'.", filterMetadataField, filterValue, e); + } + } + + String searchLower = StringUtils.isNotBlank(search) ? search.toLowerCase(Locale.ROOT) : null; + + List result = new ArrayList<>(); + for (Item item : candidates.values()) { + if (result.size() >= effectiveLimit) { + break; + } + if (!isExposed(context, item)) { + continue; + } + if (searchLower != null && !matchesSearch(item, searchLower)) { + continue; + } + if (!authorizeService.authorizeActionBoolean(context, item, Constants.READ)) { + continue; + } + result.add(item); + } + + return result; + } + + /** + * Check whether an Item's title (or name, if it has no title) contains the given (already + * lower-cased) search string. + * + * @param item the item + * @param searchLower the search string, already lower-cased + * @return {@code true} if the item's title/name contains {@code searchLower}, case-insensitively + */ + private boolean matchesSearch(Item item, String searchLower) { + String title = itemService.getMetadataFirstValue(item, "dc", "title", null, Item.ANY); + if (StringUtils.isBlank(title)) { + title = item.getName(); + } + return title != null && title.toLowerCase(Locale.ROOT).contains(searchLower); + } + + /** + * Compute a deterministic sha (revision id) for an Item, given its last-modified timestamp and the + * filename-to-md5 map of its ORIGINAL bitstreams. + * + *

The result is a 40-character lowercase hex string (SHA-1), deterministic across JVM runs and + * insensitive to the iteration order of {@code filenameToMd5}.

+ * + * @param itemUuid the item's UUID, as a string + * @param lastModifiedMillis the item's last-modified timestamp, in epoch milliseconds + * @param filenameToMd5 map of ORIGINAL bitstream filename to its MD5 checksum + * @return the computed sha, a 40-character lowercase hex string + */ + public static String computeSha(String itemUuid, long lastModifiedMillis, Map filenameToMd5) { + List pairs = new ArrayList<>(); + for (Map.Entry entry : filenameToMd5.entrySet()) { + pairs.add(entry.getKey() + ":" + entry.getValue()); + } + Collections.sort(pairs); + String joined = String.join("\n", pairs); + String input = itemUuid + "\n" + lastModifiedMillis + "\n" + joined; + return DigestUtils.sha1Hex(input); + } + + /** + * Check whether a requested revision string is valid, given the current sha of an Item. + * + * @param revision the requested revision (path parameter of the facade endpoints) + * @param currentSha the current sha of the Item, as computed by {@link #computeSha(String, long, Map)} + * @return {@code true} if {@code revision} equals {@code "main"} or equals {@code currentSha} + * (case-insensitively); {@code false} if {@code revision} is null or blank + */ + public static boolean isValidRevision(String revision, String currentSha) { + if (StringUtils.isBlank(revision)) { + return false; + } + return MAIN_REVISION.equals(revision) || revision.equalsIgnoreCase(currentSha); + } + + /** + * Compute the current sha of an Item, based on its ORIGINAL bitstreams and last-modified timestamp. + * + * @param context the DSpace context + * @param item the item + * @return the computed sha, a 40-character lowercase hex string + */ + public String computeShaForItem(Context context, Item item) { + Map originalFiles = getOriginalFiles(item); + Map filenameToMd5 = new LinkedHashMap<>(); + for (Map.Entry entry : originalFiles.entrySet()) { + filenameToMd5.put(entry.getKey(), entry.getValue().getChecksum()); + } + long lastModifiedMillis = item.getLastModified() != null ? item.getLastModified().getTime() : 0L; + String itemUuid = item.getID().toString(); + return computeSha(itemUuid, lastModifiedMillis, filenameToMd5); + } + + /** + * Collect the filename-to-bitstream map of all bitstreams in the Item's ORIGINAL bundles, iterating + * bundles then bitstreams in order. If several bitstreams share the same filename, the first one + * encountered wins. + * + * @param item the item + * @return a filename-to-bitstream map, preserving encounter order; empty if the item has no ORIGINAL + * bundle or the bundle(s) are empty + */ + public LinkedHashMap getOriginalFiles(Item item) { + LinkedHashMap result = new LinkedHashMap<>(); + List bundles = item.getBundles(Constants.CONTENT_BUNDLE_NAME); + for (Bundle bundle : bundles) { + for (Bitstream bitstream : bundle.getBitstreams()) { + result.putIfAbsent(bitstream.getName(), bitstream); + } + } + return result; + } + + /** + * Build the HuggingFace-Hub-shaped model info for an Item. + * + * @param context the DSpace context + * @param item the item + * @param sha the item's current sha, as computed by {@link #computeShaForItem(Context, Item)} + * @return the model info + */ + public HuggingFaceModelInfo toModelInfo(Context context, Item item, String sha) { + HuggingFaceModelInfo modelInfo = new HuggingFaceModelInfo(); + + String handle = item.getHandle(); + modelInfo.setId(handle); + modelInfo.setModelId(handle); + modelInfo.setAuthor(handleAuthor(handle)); + modelInfo.setSha(sha); + + LinkedHashSet tags = new LinkedHashSet<>(); + addMetadataValues(tags, itemService.getMetadataByMetadataString(item, "dc.subject")); + List languages = metadataValues(itemService.getMetadataByMetadataString(item, "dc.language.iso")); + tags.addAll(languages); + addMetadataValues(tags, itemService.getMetadataByMetadataString(item, "dc.type")); + modelInfo.setTags(new ArrayList<>(tags)); + + Map cardData = new LinkedHashMap<>(); + List rights = itemService.getMetadataByMetadataString(item, "dc.rights"); + if (!rights.isEmpty()) { + cardData.put("license", rights.get(0).getValue()); + } + if (!languages.isEmpty()) { + cardData.put("language", languages); + } + cardData.put("tags", new ArrayList<>(tags)); + modelInfo.setCardData(cardData); + + long lastModifiedMillis = item.getLastModified() != null ? item.getLastModified().getTime() : 0L; + String lastModified = Instant.ofEpochMilli(lastModifiedMillis).toString(); + modelInfo.setLastModified(lastModified); + modelInfo.setCreatedAt(resolveCreatedAt(item, lastModified)); + + List siblings = new ArrayList<>(); + for (String filename : getOriginalFiles(item).keySet()) { + siblings.add(new HuggingFaceSibling(filename)); + } + modelInfo.setSiblings(siblings); + + modelInfo.setPrivate(false); + modelInfo.setGated(false); + modelInfo.setDisabled(false); + modelInfo.setDownloads(0L); + modelInfo.setLikes(0L); + + return modelInfo; + } + + /** + * Build the HuggingFace-Hub-shaped file tree for an Item, one entry per ORIGINAL bitstream. + * + * @param context the DSpace context + * @param item the item + * @return the list of tree entries; empty if the item has no ORIGINAL bitstreams + */ + public List toTreeEntries(Context context, Item item) { + List entries = new ArrayList<>(); + for (Map.Entry entry : getOriginalFiles(item).entrySet()) { + HuggingFaceTreeEntry treeEntry = new HuggingFaceTreeEntry(); + treeEntry.setType("file"); + treeEntry.setPath(entry.getKey()); + treeEntry.setSize(entry.getValue().getSizeBytes()); + treeEntry.setOid(entry.getValue().getChecksum()); + entries.add(treeEntry); + } + return entries; + } + + /** + * Extract the author (handle prefix) from a handle. + * + * @param handle the handle, e.g. {@code "11234/1-5814"} + * @return the handle prefix, e.g. {@code "11234"}, or the handle itself if it does not contain a slash + */ + private String handleAuthor(String handle) { + if (handle == null) { + return null; + } + int slashIndex = handle.indexOf('/'); + return slashIndex >= 0 ? handle.substring(0, slashIndex) : handle; + } + + /** + * Resolve the "createdAt" timestamp of an Item from its {@code dc.date.accessioned} metadata, falling + * back to the given last-modified timestamp if the metadata is missing or unparseable. + * + * @param item the item + * @param lastModified the pre-computed, ISO-8601 last-modified timestamp to fall back on + * @return the ISO-8601 createdAt timestamp + */ + private String resolveCreatedAt(Item item, String lastModified) { + List accessioned = itemService.getMetadataByMetadataString(item, "dc.date.accessioned"); + if (accessioned.isEmpty()) { + return lastModified; + } + try { + return Instant.parse(accessioned.get(0).getValue()).toString(); + } catch (RuntimeException e) { + return lastModified; + } + } + + /** + * Extract the {@link MetadataValue#getValue()} of each entry into a new list. + * + * @param values the metadata values + * @return the extracted string values, in the same order + */ + private List metadataValues(List values) { + List result = new ArrayList<>(); + addMetadataValues(result, values); + return result; + } + + /** + * Append the {@link MetadataValue#getValue()} of each entry to the given collection. + * + * @param target the collection to append to + * @param values the metadata values + */ + private void addMetadataValues(java.util.Collection target, List values) { + for (MetadataValue value : values) { + target.add(value.getValue()); + } + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceModelInfo.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceModelInfo.java new file mode 100644 index 000000000000..f4b02c5e8ed2 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceModelInfo.java @@ -0,0 +1,312 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Plain Jackson POJO modeling the JSON shape expected by huggingface_hub's + * {@code model_info} response (as consumed by the {@code huggingface_hub} Python + * client library). This class is intentionally decoupled from the DSpace REST + * HAL/RestRepository machinery: it is a simple data holder that is serialized + * directly to JSON by the (future) HuggingFace-facade controller. + * + * @author DSpace at UFAL + */ +public class HuggingFaceModelInfo { + + private String id; + + private String modelId; + + private String author; + + private String sha; + + private String lastModified; + + private String createdAt; + + @JsonProperty("private") + private boolean isPrivate; + + private boolean gated; + + private boolean disabled; + + private long downloads; + + private long likes; + + private List tags; + + private Map cardData; + + private List siblings; + + /** + * Default no-arg constructor required for Jackson (de)serialization. + */ + public HuggingFaceModelInfo() { + } + + /** + * Get the repository id (in our case, the Item handle). + * + * @return the repository id + */ + public String getId() { + return id; + } + + /** + * Set the repository id. + * + * @param id the repository id + */ + public void setId(String id) { + this.id = id; + } + + /** + * Get the model id (in our case, the same value as {@link #getId()}). + * + * @return the model id + */ + public String getModelId() { + return modelId; + } + + /** + * Set the model id. + * + * @param modelId the model id + */ + public void setModelId(String modelId) { + this.modelId = modelId; + } + + /** + * Get the author (in our case, the handle prefix). + * + * @return the author + */ + public String getAuthor() { + return author; + } + + /** + * Set the author. + * + * @param author the author + */ + public void setAuthor(String author) { + this.author = author; + } + + /** + * Get the sha (revision) of the repository content. + * + * @return the sha + */ + public String getSha() { + return sha; + } + + /** + * Set the sha (revision) of the repository content. + * + * @param sha the sha + */ + public void setSha(String sha) { + this.sha = sha; + } + + /** + * Get the last modification timestamp, as an ISO-8601 string. + * + * @return the last modification timestamp + */ + public String getLastModified() { + return lastModified; + } + + /** + * Set the last modification timestamp. + * + * @param lastModified the last modification timestamp, as an ISO-8601 string + */ + public void setLastModified(String lastModified) { + this.lastModified = lastModified; + } + + /** + * Get the creation timestamp, as an ISO-8601 string. + * + * @return the creation timestamp + */ + public String getCreatedAt() { + return createdAt; + } + + /** + * Set the creation timestamp. + * + * @param createdAt the creation timestamp, as an ISO-8601 string + */ + public void setCreatedAt(String createdAt) { + this.createdAt = createdAt; + } + + /** + * Whether the repository is private. Always {@code false} for items exposed by this facade. + * + * @return whether the repository is private + */ + public boolean isPrivate() { + return isPrivate; + } + + /** + * Set whether the repository is private. + * + * @param isPrivate whether the repository is private + */ + public void setPrivate(boolean isPrivate) { + this.isPrivate = isPrivate; + } + + /** + * Whether the repository is gated. Always {@code false} for items exposed by this facade. + * + * @return whether the repository is gated + */ + public boolean isGated() { + return gated; + } + + /** + * Set whether the repository is gated. + * + * @param gated whether the repository is gated + */ + public void setGated(boolean gated) { + this.gated = gated; + } + + /** + * Whether the repository is disabled. Always {@code false} for items exposed by this facade. + * + * @return whether the repository is disabled + */ + public boolean isDisabled() { + return disabled; + } + + /** + * Set whether the repository is disabled. + * + * @param disabled whether the repository is disabled + */ + public void setDisabled(boolean disabled) { + this.disabled = disabled; + } + + /** + * Get the number of downloads. Always {@code 0} for items exposed by this facade. + * + * @return the number of downloads + */ + public long getDownloads() { + return downloads; + } + + /** + * Set the number of downloads. + * + * @param downloads the number of downloads + */ + public void setDownloads(long downloads) { + this.downloads = downloads; + } + + /** + * Get the number of likes. Always {@code 0} for items exposed by this facade. + * + * @return the number of likes + */ + public long getLikes() { + return likes; + } + + /** + * Set the number of likes. + * + * @param likes the number of likes + */ + public void setLikes(long likes) { + this.likes = likes; + } + + /** + * Get the tags associated with this repository. + * + * @return the tags + */ + public List getTags() { + return tags; + } + + /** + * Set the tags associated with this repository. + * + * @param tags the tags + */ + public void setTags(List tags) { + this.tags = tags; + } + + /** + * Get the card data (metadata block normally found in the model card's YAML front matter). + * + * @return the card data + */ + public Map getCardData() { + return cardData; + } + + /** + * Set the card data. + * + * @param cardData the card data + */ + public void setCardData(Map cardData) { + this.cardData = cardData; + } + + /** + * Get the list of files (siblings) belonging to this repository. + * + * @return the siblings + */ + public List getSiblings() { + return siblings; + } + + /** + * Set the list of files (siblings) belonging to this repository. + * + * @param siblings the siblings + */ + public void setSiblings(List siblings) { + this.siblings = siblings; + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceSibling.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceSibling.java new file mode 100644 index 000000000000..1ab614cad756 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceSibling.java @@ -0,0 +1,54 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +/** + * Plain Jackson POJO representing a single entry of the {@code siblings} array + * returned by the HuggingFace Hub {@code model_info} API. Each sibling corresponds + * to a single file that belongs to the "model repository" (in our case, an ORIGINAL + * bitstream of a DSpace Item). + * + * @author DSpace at UFAL + */ +public class HuggingFaceSibling { + + private String rfilename; + + /** + * Default no-arg constructor required for Jackson (de)serialization. + */ + public HuggingFaceSibling() { + } + + /** + * Construct a sibling entry for the given filename. + * + * @param rfilename the repository-relative filename + */ + public HuggingFaceSibling(String rfilename) { + this.rfilename = rfilename; + } + + /** + * Get the repository-relative filename. + * + * @return the filename + */ + public String getRfilename() { + return rfilename; + } + + /** + * Set the repository-relative filename. + * + * @param rfilename the filename + */ + public void setRfilename(String rfilename) { + this.rfilename = rfilename; + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceTreeEntry.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceTreeEntry.java new file mode 100644 index 000000000000..e7e79d31ac50 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceTreeEntry.java @@ -0,0 +1,105 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +/** + * Plain Jackson POJO representing a single entry returned by the HuggingFace Hub + * "tree" endpoint (e.g. {@code GET /api/models/{repo_id}/tree/{revision}}). Each entry + * describes one file present in the repository (in our case, an ORIGINAL bitstream of + * a DSpace Item). + * + * @author DSpace at UFAL + */ +public class HuggingFaceTreeEntry { + + private String type; + + private String path; + + private long size; + + private String oid; + + /** + * Default no-arg constructor required for Jackson (de)serialization. + */ + public HuggingFaceTreeEntry() { + } + + /** + * Get the entry type, e.g. {@code "file"}. + * + * @return the entry type + */ + public String getType() { + return type; + } + + /** + * Set the entry type. + * + * @param type the entry type + */ + public void setType(String type) { + this.type = type; + } + + /** + * Get the repository-relative path of the file. + * + * @return the file path + */ + public String getPath() { + return path; + } + + /** + * Set the repository-relative path of the file. + * + * @param path the file path + */ + public void setPath(String path) { + this.path = path; + } + + /** + * Get the file size in bytes. + * + * @return the file size in bytes + */ + public long getSize() { + return size; + } + + /** + * Set the file size in bytes. + * + * @param size the file size in bytes + */ + public void setSize(long size) { + this.size = size; + } + + /** + * Get the object id (checksum) of the file, as reported by the git/LFS-like tree API. + * + * @return the object id + */ + public String getOid() { + return oid; + } + + /** + * Set the object id (checksum) of the file. + * + * @param oid the object id + */ + public void setOid(String oid) { + this.oid = oid; + } +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java new file mode 100644 index 000000000000..87427d514954 --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java @@ -0,0 +1,434 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.matchesPattern; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertEquals; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.head; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.io.InputStream; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.codec.CharEncoding; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.IOUtils; +import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.builder.BitstreamBuilder; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.builder.ResourcePolicyBuilder; +import org.dspace.content.Bitstream; +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.content.service.ItemService; +import org.dspace.core.Constants; +import org.dspace.services.ConfigurationService; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.web.servlet.MvcResult; + +/** + * Integration tests for {@link ClarinHuggingFaceController}. + */ +public class ClarinHuggingFaceControllerIT extends AbstractControllerIntegrationTest { + + private static final String ENDPOINT_BASE = "/api/hf/api/models"; + + private static final String ZERO_REVISION = "0000000000000000000000000000000000000000"; + + private static final String MODEL_BIN_CONTENT = "hello model"; + + private static final String CONFIG_JSON_CONTENT = "config content"; + + @Autowired + private ConfigurationService configurationService; + + @Autowired + private AuthorizeService authorizeService; + + @Autowired + private ItemService itemService; + + private Item exposedItem; + + private Item nonExposedItem; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + + configurationService.setProperty("hf.api.enabled", true); + configurationService.setProperty("hf.api.filter.metadata-field", "dc.type"); + configurationService.setProperty("hf.api.filter.value", "machineLearningModel"); + + context.turnOffAuthorisationSystem(); + + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection collection = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + + exposedItem = ItemBuilder.createItem(context, collection) + .withTitle("A Machine Learning Model") + .withType("machineLearningModel") + .withSubject("nlp") + .withSubject("classification") + .build(); + try (InputStream is = IOUtils.toInputStream(MODEL_BIN_CONTENT, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, exposedItem, is) + .withName("model.bin") + .withMimeType("application/octet-stream") + .build(); + } + try (InputStream is = IOUtils.toInputStream(CONFIG_JSON_CONTENT, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, exposedItem, is) + .withName("config.json") + .withMimeType("application/json") + .build(); + } + + nonExposedItem = ItemBuilder.createItem(context, collection) + .withTitle("A Plain Item") + .build(); + + context.restoreAuthSystemState(); + } + + @Test + public void modelInfoReturnsExpectedShape() throws Exception { + String[] handleParts = exposedItem.getHandle().split("/"); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1])) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.id", is(exposedItem.getHandle()))) + .andExpect(jsonPath("$.sha", matchesPattern("^[0-9a-f]{40}$"))) + .andExpect(jsonPath("$.private", is(false))) + .andExpect(jsonPath("$.gated", is(false))) + .andExpect(jsonPath("$.downloads", is(0))) + .andExpect(jsonPath("$.siblings", hasSize(2))) + .andExpect(jsonPath("$.siblings[*].rfilename", hasItem("model.bin"))) + .andExpect(jsonPath("$.siblings[*].rfilename", hasItem("config.json"))) + .andExpect(jsonPath("$.tags", hasItem("nlp"))) + .andExpect(jsonPath("$.tags", hasItem("classification"))) + .andExpect(jsonPath("$.tags", hasItem("machineLearningModel"))); + } + + @Test + public void revisionMainIsValidAndZeroRevisionIsNotFound() throws Exception { + String[] handleParts = exposedItem.getHandle().split("/"); + String base = ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1]; + + getClient().perform(get(base + "/revision/main")) + .andExpect(status().isOk()); + + getClient().perform(get(base + "/revision/" + ZERO_REVISION)) + .andExpect(status().isNotFound()) + .andExpect(header().string("X-Error-Code", "RevisionNotFound")); + } + + @Test + public void treeMainReturnsFileEntriesWithCorrectSizeAndOid() throws Exception { + String[] handleParts = exposedItem.getHandle().split("/"); + + MvcResult result = getClient() + .perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/tree/main")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(2))) + .andExpect(jsonPath("$[*].type", everyItem(is("file")))) + .andReturn(); + + JsonNode entries = new ObjectMapper().readTree(result.getResponse().getContentAsString()); + Map byPath = new HashMap<>(); + for (JsonNode entry : entries) { + byPath.put(entry.get("path").asText(), entry); + } + + assertEquals(MODEL_BIN_CONTENT.length(), byPath.get("model.bin").get("size").asLong()); + assertEquals(DigestUtils.md5Hex(MODEL_BIN_CONTENT), byPath.get("model.bin").get("oid").asText()); + assertEquals(CONFIG_JSON_CONTENT.length(), byPath.get("config.json").get("size").asLong()); + assertEquals(DigestUtils.md5Hex(CONFIG_JSON_CONTENT), byPath.get("config.json").get("oid").asText()); + } + + @Test + public void treeAcceptsCurrentSha() throws Exception { + String[] handleParts = exposedItem.getHandle().split("/"); + String base = ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1]; + + MvcResult result = getClient().perform(get(base)) + .andExpect(status().isOk()) + .andReturn(); + String content = result.getResponse().getContentAsString(); + JsonNode json = new ObjectMapper().readTree(content); + String sha = json.get("sha").asText(); + + getClient().perform(get(base + "/tree/" + sha)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(2))); + } + + @Test + public void nonExposedItemIsNotFound() throws Exception { + String[] handleParts = nonExposedItem.getHandle().split("/"); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1])) + .andExpect(status().isNotFound()) + .andExpect(header().string("X-Error-Code", "RepoNotFound")); + } + + @Test + public void listModelsReturnsOnlyExposedItems() throws Exception { + context.turnOffAuthorisationSystem(); + Collection collection = exposedItem.getOwningCollection(); + Item secondExposedItem = ItemBuilder.createItem(context, collection) + .withTitle("Another Machine Learning Model") + .withType("machineLearningModel") + .build(); + context.restoreAuthSystemState(); + + getClient().perform(get(ENDPOINT_BASE)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$[*].id", hasItem(exposedItem.getHandle()))) + .andExpect(jsonPath("$[*].id", hasItem(secondExposedItem.getHandle()))) + .andExpect(jsonPath("$[*].id", not(hasItem(nonExposedItem.getHandle())))); + } + + @Test + public void listModelsExcludesItemsWithoutReadAccess() throws Exception { + context.turnOffAuthorisationSystem(); + Collection collection = exposedItem.getOwningCollection(); + Item restrictedExposedItem = ItemBuilder.createItem(context, collection) + .withTitle("Restricted Machine Learning Model") + .withType("machineLearningModel") + .build(); + // Remove all read policies from the item and grant READ to admin only, so the item is still + // "exposed" (passes the HuggingFace filter) but not readable by an anonymous user. + authorizeService.removeAllPolicies(context, restrictedExposedItem); + ResourcePolicyBuilder.createResourcePolicy(context, admin, null) + .withDspaceObject(restrictedExposedItem) + .withAction(Constants.READ) + .build(); + context.restoreAuthSystemState(); + + // Anonymous user cannot read the restricted item, so it must not appear in the listing. + getClient().perform(get(ENDPOINT_BASE)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$[*].id", hasItem(exposedItem.getHandle()))) + .andExpect(jsonPath("$[*].id", not(hasItem(restrictedExposedItem.getHandle())))); + + // The admin can read it, so it must appear in the listing. + String adminToken = getAuthToken(admin.getEmail(), password); + getClient(adminToken).perform(get(ENDPOINT_BASE)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$[*].id", hasItem(restrictedExposedItem.getHandle()))); + } + + @Test + public void listModelsSupportsSearchAndLimit() throws Exception { + context.turnOffAuthorisationSystem(); + Collection collection = exposedItem.getOwningCollection(); + ItemBuilder.createItem(context, collection) + .withTitle("Second Model") + .withType("machineLearningModel") + .build(); + context.restoreAuthSystemState(); + + getClient().perform(get(ENDPOINT_BASE).param("search", "Machine Learning")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(1))) + .andExpect(jsonPath("$[0].id", is(exposedItem.getHandle()))); + + getClient().perform(get(ENDPOINT_BASE).param("limit", "1")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(1))); + } + + @Test + public void listModelsDisabledFacadeReturnsNotFound() throws Exception { + configurationService.setProperty("hf.api.enabled", false); + + getClient().perform(get(ENDPOINT_BASE)) + .andExpect(status().isNotFound()); + } + + @Test + public void disabledFacadeReturnsNotFound() throws Exception { + configurationService.setProperty("hf.api.enabled", false); + String[] handleParts = exposedItem.getHandle().split("/"); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1])) + .andExpect(status().isNotFound()); + } + + @Test + public void withdrawnItemIsNotFound() throws Exception { + String[] handleParts = exposedItem.getHandle().split("/"); + + context.turnOffAuthorisationSystem(); + itemService.withdraw(context, exposedItem); + context.restoreAuthSystemState(); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1])) + .andExpect(status().isNotFound()) + .andExpect(header().string("X-Error-Code", "RepoNotFound")); + } + + @Test + public void whoAmIAlwaysReturnsUnauthorized() throws Exception { + getClient().perform(get("/api/hf/api/whoami-v2")) + .andExpect(status().isUnauthorized()); + } + + @Test + public void resolveHeadReturnsMetadataHeaders() throws Exception { + String handle = exposedItem.getHandle(); + String sha = fetchModelSha(handle); + + getClient().perform(head("/api/hf/" + handle + "/resolve/main/model.bin")) + .andExpect(status().isOk()) + .andExpect(header().string("X-Repo-Commit", sha)) + .andExpect(header().string("ETag", "\"" + DigestUtils.md5Hex(MODEL_BIN_CONTENT) + "\"")) + .andExpect(header().longValue("Content-Length", MODEL_BIN_CONTENT.length())) + .andExpect(header().string("Accept-Ranges", "bytes")) + .andExpect(header().exists("X-Linked-Etag")); + } + + @Test + public void resolveGetStreamsFileContent() throws Exception { + String handle = exposedItem.getHandle(); + + getClient().perform(get("/api/hf/" + handle + "/resolve/main/model.bin")) + .andExpect(status().isOk()) + .andExpect(header().exists("X-Repo-Commit")) + .andExpect(content().bytes(MODEL_BIN_CONTENT.getBytes())); + } + + @Test + public void resolveGetSupportsRangeRequests() throws Exception { + String handle = exposedItem.getHandle(); + + getClient().perform(get("/api/hf/" + handle + "/resolve/main/model.bin") + .header("Range", "bytes=1-3")) + .andExpect(status().is(206)) + // The Content-Length must match the requested range + .andExpect(header().longValue("Content-Length", 3)) + // The server should indicate we support Range requests + .andExpect(header().string("Accept-Ranges", "bytes")) + // The ETag has to be based on the checksum + .andExpect(header().string("ETag", "\"" + DigestUtils.md5Hex(MODEL_BIN_CONTENT) + "\"")) + // The response should give us details about the range + .andExpect(header().string("Content-Range", "bytes 1-3/" + MODEL_BIN_CONTENT.length())) + // We only expect the bytes 1, 2 and 3 + .andExpect(content().bytes(MODEL_BIN_CONTENT.substring(1, 4).getBytes())); + } + + @Test + public void resolveAcceptsCurrentShaAndRejectsUnknownRevision() throws Exception { + String handle = exposedItem.getHandle(); + String sha = fetchModelSha(handle); + + getClient().perform(get("/api/hf/" + handle + "/resolve/" + sha + "/model.bin")) + .andExpect(status().isOk()) + .andExpect(content().bytes(MODEL_BIN_CONTENT.getBytes())); + + getClient().perform(get("/api/hf/" + handle + "/resolve/" + ZERO_REVISION + "/model.bin")) + .andExpect(status().isNotFound()) + .andExpect(header().string("X-Error-Code", "RevisionNotFound")); + } + + @Test + public void resolveRestrictedBitstreamReturnsUnauthorizedOrForbidden() throws Exception { + String handle = exposedItem.getHandle(); + + context.turnOffAuthorisationSystem(); + Bitstream restricted; + try (InputStream is = IOUtils.toInputStream("secret content", CharEncoding.UTF_8)) { + restricted = BitstreamBuilder.createBitstream(context, exposedItem, is) + .withName("secret.bin") + .withMimeType("application/octet-stream") + .build(); + } + // Remove all read policies from the bitstream and add a read policy only for admin + authorizeService.removeAllPolicies(context, restricted); + ResourcePolicyBuilder.createResourcePolicy(context, admin, null) + .withDspaceObject(restricted) + .withAction(Constants.READ) + .build(); + context.restoreAuthSystemState(); + + // Anonymous user should get 401 + getClient().perform(get("/api/hf/" + handle + "/resolve/main/secret.bin")) + .andExpect(status().isUnauthorized()); + + // Authenticated non-admin user should get 403 + String token = getAuthToken(eperson.getEmail(), password); + getClient(token).perform(get("/api/hf/" + handle + "/resolve/main/secret.bin")) + .andExpect(status().isForbidden()); + } + + @Test + public void resolveMissingFileReturnsEntryNotFound() throws Exception { + String handle = exposedItem.getHandle(); + + getClient().perform(get("/api/hf/" + handle + "/resolve/main/nope.bin")) + .andExpect(status().isNotFound()) + .andExpect(header().string("X-Error-Code", "EntryNotFound")); + } + + @Test + public void resolveSupportsUrlEncodedFilenames() throws Exception { + String handle = exposedItem.getHandle(); + String encodedFileContent = "encoded filename content"; + + context.turnOffAuthorisationSystem(); + try (InputStream is = IOUtils.toInputStream(encodedFileContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, exposedItem, is) + .withName("my model (v2).bin") + .withMimeType("application/octet-stream") + .build(); + } + context.restoreAuthSystemState(); + + getClient().perform(get(URI.create("/api/hf/" + handle + "/resolve/main/my%20model%20%28v2%29.bin"))) + .andExpect(status().isOk()) + .andExpect(content().bytes(encodedFileContent.getBytes())); + } + + /** + * Fetch the current sha of a model repository from its model-info JSON. + * + * @param handle the item handle + * @return the sha reported by the model-info endpoint + */ + private String fetchModelSha(String handle) throws Exception { + MvcResult result = getClient().perform(get(ENDPOINT_BASE + "/" + handle)) + .andExpect(status().isOk()) + .andReturn(); + JsonNode json = new ObjectMapper().readTree(result.getResponse().getContentAsString()); + return json.get("sha").asText(); + } +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/hf/ClarinHuggingFaceServiceTest.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/hf/ClarinHuggingFaceServiceTest.java new file mode 100644 index 000000000000..3f59d33eaecd --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/hf/ClarinHuggingFaceServiceTest.java @@ -0,0 +1,169 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.Test; + +/** + * Plain JUnit 4 unit tests (no Spring context) for the pure static helper methods of + * {@link ClarinHuggingFaceService}: {@link ClarinHuggingFaceService#computeSha(String, long, Map)} and + * {@link ClarinHuggingFaceService#isValidRevision(String, String)}. + * + * @author DSpace at UFAL + */ +public class ClarinHuggingFaceServiceTest { + + private static final String UUID_1 = "11111111-1111-1111-1111-111111111111"; + + private static final String UUID_2 = "22222222-2222-2222-2222-222222222222"; + + private static final long LAST_MODIFIED_1 = 1_700_000_000_000L; + + private static final long LAST_MODIFIED_2 = 1_800_000_000_000L; + + /** + * Build a sample filename-to-md5 map. + * + * @return a map with two entries + */ + private Map sampleFiles() { + Map files = new LinkedHashMap<>(); + files.put("model.bin", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + files.put("config.json", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + return files; + } + + @Test + public void testComputeShaReturns40CharLowercaseHex() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertTrue("sha should match [0-9a-f]{40} but was: " + sha, sha.matches("[0-9a-f]{40}")); + } + + @Test + public void testComputeShaIsDeterministic() { + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertEquals(sha1, sha2); + } + + @Test + public void testComputeShaIsInsensitiveToMapInsertionOrder() { + Map hashMapOrder = new HashMap<>(); + hashMapOrder.put("model.bin", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + hashMapOrder.put("config.json", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + hashMapOrder.put("tokenizer.json", "cccccccccccccccccccccccccccccccc"); + + Map reversedLinkedOrder = new LinkedHashMap<>(); + reversedLinkedOrder.put("tokenizer.json", "cccccccccccccccccccccccccccccccc"); + reversedLinkedOrder.put("config.json", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + reversedLinkedOrder.put("model.bin", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + + String shaFromHashMap = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, hashMapOrder); + String shaFromReversedLinkedMap = + ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, reversedLinkedOrder); + + assertEquals(shaFromHashMap, shaFromReversedLinkedMap); + } + + @Test + public void testComputeShaChangesWhenUuidChanges() { + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_2, LAST_MODIFIED_1, sampleFiles()); + assertNotEquals(sha1, sha2); + } + + @Test + public void testComputeShaChangesWhenLastModifiedChanges() { + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_2, sampleFiles()); + assertNotEquals(sha1, sha2); + } + + @Test + public void testComputeShaChangesWhenFilenameChanges() { + Map files = sampleFiles(); + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, files); + + Map renamed = new LinkedHashMap<>(); + renamed.put("model-renamed.bin", files.get("model.bin")); + renamed.put("config.json", files.get("config.json")); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, renamed); + + assertNotEquals(sha1, sha2); + } + + @Test + public void testComputeShaChangesWhenMd5Changes() { + Map files = sampleFiles(); + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, files); + + Map changed = new LinkedHashMap<>(files); + changed.put("model.bin", "ffffffffffffffffffffffffffffffff"); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, changed); + + assertNotEquals(sha1, sha2); + } + + @Test + public void testComputeShaChangesWhenFileAdded() { + Map files = sampleFiles(); + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, files); + + Map withExtra = new LinkedHashMap<>(files); + withExtra.put("extra.txt", "dddddddddddddddddddddddddddddddd"); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, withExtra); + + assertNotEquals(sha1, sha2); + } + + @Test + public void testIsValidRevisionMainIsAlwaysValid() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertTrue(ClarinHuggingFaceService.isValidRevision("main", sha)); + } + + @Test + public void testIsValidRevisionMatchingShaIsValid() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertTrue(ClarinHuggingFaceService.isValidRevision(sha, sha)); + } + + @Test + public void testIsValidRevisionMatchingShaIsCaseInsensitive() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertTrue(ClarinHuggingFaceService.isValidRevision(sha.toUpperCase(), sha)); + } + + @Test + public void testIsValidRevisionDifferentShaIsInvalid() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + String otherSha = ClarinHuggingFaceService.computeSha(UUID_2, LAST_MODIFIED_2, sampleFiles()); + assertFalse(ClarinHuggingFaceService.isValidRevision(otherSha, sha)); + } + + @Test + public void testIsValidRevisionNullIsInvalid() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertFalse(ClarinHuggingFaceService.isValidRevision(null, sha)); + } + + @Test + public void testIsValidRevisionEmptyIsInvalid() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertFalse(ClarinHuggingFaceService.isValidRevision("", sha)); + } +} diff --git a/dspace/config/clarin-dspace.cfg b/dspace/config/clarin-dspace.cfg index a481beb856aa..312f2e06f3bb 100644 --- a/dspace/config/clarin-dspace.cfg +++ b/dspace/config/clarin-dspace.cfg @@ -344,3 +344,25 @@ user.registration = false # This option allows submitter to skip file upload step in the submission process webui.submit.upload.required = false + +#------------------------------------------------------------------# +#-------------------------HUGGINGFACE FACADE------------------------# +#------------------------------------------------------------------# +# HuggingFace-Hub-compatible read-only API facade, allowing tools built on the +# huggingface_hub Python client (e.g. `from_pretrained`) to resolve DSpace items as +# "models". An item is exposed through the facade only if it passes the exposure +# filter below (metadata field/value match) or belongs to one of the listed +# collections; if neither is configured, no items are exposed. + +# Master switch for the HuggingFace-Hub-compatible facade. +hf.api.enabled = false + +# Metadata field used to select items exposed through the HuggingFace facade. +hf.api.filter.metadata-field = dc.type + +# Value that `hf.api.filter.metadata-field` must equal for an item to be exposed. +hf.api.filter.value = machineLearningModel + +# Comma-separated list of collection handles whose items are exposed through the +# HuggingFace facade, regardless of the metadata filter above. +hf.api.exposed.collections =