From 108091c0361f1b02d4acc620dad4cf6a05ff50e2 Mon Sep 17 00:00:00 2001 From: Ondrej Kosarko Date: Wed, 8 Jul 2026 16:57:21 +0200 Subject: [PATCH 1/2] Feature: HuggingFace-Hub-compatible API facade for CLARIN-DSpace Implement a REST API that wraps DSpace items as HuggingFace-compatible model repositories, allowing HuggingFace client tools (transformers.from_pretrained, huggingface_hub snapshot_download, hf CLI) to consume models stored in DSpace by setting HF_ENDPOINT environment variable. Implementation: - New @RestController at /api/hf with routes: * GET /api/models/{handle} - model metadata (sha, files, tags) * GET /api/models/{handle}/revision/{revision} - revision-specific metadata * GET /api/models/{handle}/tree/{revision} - file listing * GET|HEAD /{handle}/resolve/{revision}/{filename} - file download with Range/206 support * GET /api/models?search=&limit= - model listing with filtering * GET /api/whoami-v2 - authentication status (401 phase-1) - Exposure filter: items must match metadata-field filter (default dc.type=machineLearningModel) or owning collection must be in hf.api.exposed.collections; fail-closed by default - SHA computation: deterministic 40-hex SHA-1 from item UUID + last-modified + bitstream MD5s, enables client-side caching and resumable downloads - File download semantics: via BitstreamResource machinery (Range/206/304/412 support), UsageEvent/Matomo tracking, S3 direct-download redirect when enabled - Authorization: existing CLARIN license-gating and READ policies apply; anonymous access only for non-gated bitstreams (phase 1) Configuration (default disabled): - hf.api.enabled (boolean, default false) - hf.api.filter.metadata-field (default dc.type) - hf.api.filter.value (default machineLearningModel) - hf.api.exposed.collections (handle list) Validation: - 14 unit tests (service logic, sha determinism, revision matching) - 18 integration tests (endpoints, auth, error codes, Range requests, encoded filenames) - Full unit suite: 1,538 tests, 0 failures - Full IT suite: 69 tests, 0 failures - Live E2E: real huggingface_hub client (snapshot_download, cache semantics verified) --- .../rest/hf/ClarinHuggingFaceController.java | 625 ++++++++++++++++++ .../app/rest/hf/ClarinHuggingFaceService.java | 498 ++++++++++++++ .../app/rest/hf/HuggingFaceModelInfo.java | 312 +++++++++ .../app/rest/hf/HuggingFaceSibling.java | 54 ++ .../app/rest/hf/HuggingFaceTreeEntry.java | 105 +++ .../rest/ClarinHuggingFaceControllerIT.java | 404 +++++++++++ .../rest/hf/ClarinHuggingFaceServiceTest.java | 169 +++++ dspace/config/clarin-dspace.cfg | 22 + 8 files changed, 2189 insertions(+) create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceModelInfo.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceSibling.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceTreeEntry.java create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/hf/ClarinHuggingFaceServiceTest.java diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java new file mode 100644 index 000000000000..169b1b4ac94e --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java @@ -0,0 +1,625 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.InternalServerErrorException; + +import org.apache.catalina.connector.ClientAbortException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.tomcat.util.http.FastHttpDateFormat; +import org.dspace.app.rest.utils.ContextUtil; +import org.dspace.app.rest.utils.HttpHeadersInitializer; +import org.dspace.app.statistics.clarin.ClarinMatomoBitstreamTracker; +import org.dspace.authorize.AuthorizeException; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.content.Bitstream; +import org.dspace.content.BitstreamFormat; +import org.dspace.content.Item; +import org.dspace.core.Constants; +import org.dspace.core.Context; +import org.dspace.eperson.EPerson; +import org.dspace.services.ConfigurationService; +import org.dspace.services.EventService; +import org.dspace.storage.bitstore.S3BitStoreService; +import org.dspace.storage.bitstore.service.S3DirectDownloadService; +import org.dspace.usage.UsageEvent; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.util.UriUtils; + +/** + * HuggingFace-Hub-compatible facade controller (read-only). + * + *

Exposes a subset of the {@code huggingface_hub} REST API shape (model info, tree listing, file + * download/"resolve" and {@code whoami-v2}) backed by {@link ClarinHuggingFaceService}, so that DSpace + * Items flagged as "machine learning models" (see the {@code hf.api.*} configuration keys) can be + * consumed directly by HuggingFace-Hub-aware tooling (e.g. the {@code huggingface_hub} Python client) + * using the Item handle as the repository id.

+ * + *

The file-download ({@code resolve}) route streams (or, when configured for S3 direct download, + * redirects to) the ORIGINAL bitstreams of an exposed Item.

+ * + *

Note: {@code @PreAuthorize} is not used because authorization depends on the resolved Item (looked + * up by handle), not on a UUID path variable. Authorization is explicitly checked via + * {@link AuthorizeService#authorizeAction} after the Item is resolved, mirroring + * {@code BitstreamByHandleRestController}.

+ * + * @author DSpace at UFAL + */ +@RestController +@RequestMapping("/api/hf") +public class ClarinHuggingFaceController { + + private static final Logger log = LogManager.getLogger(ClarinHuggingFaceController.class); + + private static final String ERROR_CODE_HEADER = "X-Error-Code"; + + private static final String REPO_COMMIT_HEADER = "X-Repo-Commit"; + + private static final String LINKED_ETAG_HEADER = "X-Linked-Etag"; + + private static final String LINKED_SIZE_HEADER = "X-Linked-Size"; + + private static final String RESOLVE_MARKER = "/resolve/"; + + // Most file systems are configured to use block sizes of 4096 or 8192 and our buffer should be a + // multiple of that (mirrors BitstreamRestController / BitstreamByHandleRestController). + private static final int BUFFER_SIZE = 4096 * 10; + + @Autowired + private ClarinHuggingFaceService huggingFaceService; + + @Autowired + private AuthorizeService authorizeService; + + @Autowired + private ConfigurationService configurationService; + + @Autowired + private EventService eventService; + + @Autowired + private ClarinMatomoBitstreamTracker matomoBitstreamTracker; + + @Autowired + private S3DirectDownloadService s3DirectDownloadService; + + @Autowired + private S3BitStoreService s3BitStoreService; + + /** + * Get the model info of an exposed Item, at the implicit "main" revision. + * + * @param prefix the handle prefix (e.g. "11234") + * @param suffix the handle suffix (e.g. "1-5814") + * @param request the HTTP request + * @return the model info, or an error response + * @throws SQLException if a database error occurs + */ + @GetMapping("/api/models/{prefix}/{suffix}") + public ResponseEntity modelInfo(@PathVariable String prefix, @PathVariable String suffix, + HttpServletRequest request) throws SQLException { + if (!huggingFaceService.isEnabled()) { + return ResponseEntity.notFound().build(); + } + + Context context = ContextUtil.obtainContext(request); + try { + ResolvedRepository repository = resolveAuthorizedRepository(context, prefix, suffix); + HuggingFaceModelInfo modelInfo = + huggingFaceService.toModelInfo(context, repository.item, repository.sha); + context.complete(); + return ResponseEntity.ok(modelInfo); + } catch (HuggingFaceApiException e) { + return e.toResponse(); + } + } + + /** + * Get the model info of an exposed Item, at a specific (validated) revision. + * + * @param prefix the handle prefix (e.g. "11234") + * @param suffix the handle suffix (e.g. "1-5814") + * @param revision the requested revision; must be {@code "main"} or the item's current sha + * @param request the HTTP request + * @return the model info, or an error response + * @throws SQLException if a database error occurs + */ + @GetMapping("/api/models/{prefix}/{suffix}/revision/{revision}") + public ResponseEntity modelInfoAtRevision(@PathVariable String prefix, @PathVariable String suffix, + @PathVariable String revision, HttpServletRequest request) throws SQLException { + if (!huggingFaceService.isEnabled()) { + return ResponseEntity.notFound().build(); + } + + Context context = ContextUtil.obtainContext(request); + try { + ResolvedRepository repository = resolveAuthorizedRepository(context, prefix, suffix); + requireValidRevision(revision, repository.sha); + HuggingFaceModelInfo modelInfo = + huggingFaceService.toModelInfo(context, repository.item, repository.sha); + context.complete(); + return ResponseEntity.ok(modelInfo); + } catch (HuggingFaceApiException e) { + return e.toResponse(); + } + } + + /** + * Get the flat file tree of an exposed Item, at a specific (validated) revision. + * + * @param prefix the handle prefix (e.g. "11234") + * @param suffix the handle suffix (e.g. "1-5814") + * @param revision the requested revision; must be {@code "main"} or the item's current sha + * @param recursive accepted and ignored (single-page flat listing regardless of its value) + * @param request the HTTP request + * @return the flat list of tree entries, or an error response + * @throws SQLException if a database error occurs + */ + @GetMapping("/api/models/{prefix}/{suffix}/tree/{revision}") + public ResponseEntity tree(@PathVariable String prefix, @PathVariable String suffix, + @PathVariable String revision, + @RequestParam(required = false) String recursive, + HttpServletRequest request) throws SQLException { + if (!huggingFaceService.isEnabled()) { + return ResponseEntity.notFound().build(); + } + + Context context = ContextUtil.obtainContext(request); + try { + ResolvedRepository repository = resolveAuthorizedRepository(context, prefix, suffix); + requireValidRevision(revision, repository.sha); + List entries = huggingFaceService.toTreeEntries(context, repository.item); + context.complete(); + return ResponseEntity.ok(entries); + } catch (HuggingFaceApiException e) { + return e.toResponse(); + } + } + + /** + * List exposed Items ("models"), HuggingFace-Hub {@code GET /api/models} style. + * + *

This is a metadata-only listing of publicly exposed items: unlike the model-info/tree/resolve + * endpoints, no per-item {@code READ} authorization check is performed, since only items that already + * pass {@link ClarinHuggingFaceService#isExposed(Context, Item)} (fail-closed, public-by-configuration) + * are ever returned.

+ * + * @param search an optional, case-insensitive substring to match against the item title + * @param limit the maximum number of items to return (default 20, capped at 100) + * @param request the HTTP request + * @return the matching model-info objects, as a JSON array + * @throws SQLException if a database error occurs + */ + @GetMapping("/api/models") + public ResponseEntity listModels(@RequestParam(required = false) String search, + @RequestParam(required = false, defaultValue = "20") int limit, + HttpServletRequest request) throws SQLException { + if (!huggingFaceService.isEnabled()) { + return ResponseEntity.notFound().build(); + } + + Context context = ContextUtil.obtainContext(request); + List models = new ArrayList<>(); + for (Item item : huggingFaceService.findExposedItems(context, search, limit)) { + String sha = huggingFaceService.computeShaForItem(context, item); + models.add(huggingFaceService.toModelInfo(context, item, sha)); + } + context.complete(); + return ResponseEntity.ok(models); + } + + /** + * Placeholder for the HuggingFace Hub {@code whoami-v2} endpoint. Always responds with 401, since + * this facade does not (yet) support Bearer-token authentication. + * + * @return a 401 response with a small JSON error body + */ + @GetMapping("/api/whoami-v2") + public ResponseEntity whoAmI() { + // TODO: phase 2 - map Authorization: Bearer to CLARIN Personal Access Token here + return ResponseEntity.status(HttpStatus.UNAUTHORIZED) + .body(Collections.singletonMap("error", "Invalid user token")); + } + + /** + * Download (GET) or probe (HEAD) a single ORIGINAL bitstream of an exposed Item, HuggingFace-Hub + * "resolve" style: {@code /{prefix}/{suffix}/resolve/{revision}/{filename}}. + * + *

The filename is everything after the revision path segment (it may contain slashes and + * URL-encoded characters, decoded once). HEAD requests are always answered locally with the metadata + * headers ({@code X-Repo-Commit}, {@code ETag}, {@code X-Linked-Etag}, {@code Content-Length}, + * {@code X-Linked-Size}, {@code Accept-Ranges}) and an empty body; they are never redirected. GET + * requests are either redirected (302) to an S3 presigned URL when direct S3 download is enabled, or + * streamed with full Range/conditional-request support.

+ * + * @param prefix the handle prefix (e.g. "11234") + * @param suffix the handle suffix (e.g. "1-5814") + * @param revision the requested revision; must be {@code "main"} or the item's current sha + * @param request the HTTP request + * @param response the HTTP response + * @return the response entity (file content, redirect or error), or {@code null} when the response + * has already been committed (e.g. 304/412 shortcuts or a client abort) + * @throws SQLException if a database error occurs + * @throws IOException if an I/O error occurs while initialising the response + */ + @RequestMapping(method = {RequestMethod.GET, RequestMethod.HEAD}, + value = "/{prefix}/{suffix}/resolve/{revision}/**") + public ResponseEntity resolveFile(@PathVariable String prefix, @PathVariable String suffix, + @PathVariable String revision, HttpServletRequest request, HttpServletResponse response) + throws SQLException, IOException { + if (!huggingFaceService.isEnabled()) { + return ResponseEntity.notFound().build(); + } + + Context context = ContextUtil.obtainContext(request); + try { + if (Objects.isNull(context)) { + log.error("Cannot obtain the context from the request."); + throw new HuggingFaceApiException(HttpStatus.INTERNAL_SERVER_ERROR, null, "Internal error."); + } + + String handle = prefix + "/" + suffix; + Item item = huggingFaceService.resolveExposedItem(context, handle); + if (item == null) { + throw new HuggingFaceApiException(HttpStatus.NOT_FOUND, "RepoNotFound", "Repository not found"); + } + + String sha = huggingFaceService.computeShaForItem(context, item); + requireValidRevision(revision, sha); + + String filename = extractResolveFilename(request); + if (StringUtils.isBlank(filename)) { + throw new HuggingFaceApiException(HttpStatus.NOT_FOUND, "EntryNotFound", "Entry not found"); + } + + Bitstream bitstream = huggingFaceService.getOriginalFiles(item).get(filename); + if (bitstream == null) { + throw new HuggingFaceApiException(HttpStatus.NOT_FOUND, "EntryNotFound", "Entry not found"); + } + + try { + // Authorization is checked explicitly here (not via @PreAuthorize) because the bitstream + // identity is resolved from handle + filename, not from a UUID path variable. + authorizeService.authorizeAction(context, bitstream, Constants.READ); + } catch (AuthorizeException e) { + // TODO: phase 2 - Bearer/PAT auth: map Authorization: Bearer to a CLARIN Personal Access + // Token and re-check authorization with the resolved EPerson before falling back to the + // anonymous 401/403 distinction below. + log.warn("Unauthorized access to file '{}' of HuggingFace repository '{}'.", filename, handle); + HttpStatus status = context.getCurrentUser() == null + ? HttpStatus.UNAUTHORIZED : HttpStatus.FORBIDDEN; + String landingPage = configurationService.getProperty("dspace.ui.url") + "/handle/" + handle; + throw new HuggingFaceApiException(status, null, + "You are not authorized to download this file. Please visit the item landing page at " + + landingPage + " to request access."); + } + + return serveBitstream(context, request, response, item, bitstream, sha); + } catch (HuggingFaceApiException e) { + return e.toResponse(); + } + } + + /** + * Serve a resolved and authorized bitstream: HEAD locally, GET via an S3 presigned-URL redirect when + * direct S3 download is enabled, or by streaming the content otherwise. + * + * @param context the DSpace context + * @param request the HTTP request + * @param response the HTTP response + * @param item the owning item (source of {@code Last-Modified}) + * @param bitstream the bitstream to serve + * @param sha the item's current sha (sent as {@code X-Repo-Commit}) + * @return the response entity, or {@code null} when the response has already been committed + * @throws SQLException if a database error occurs + * @throws IOException if an I/O error occurs while initialising the response + */ + private ResponseEntity serveBitstream(Context context, HttpServletRequest request, + HttpServletResponse response, Item item, Bitstream bitstream, String sha) + throws SQLException, IOException { + BitstreamFormat format = bitstream.getFormat(context); + String mimetype = format != null ? format.getMIMEType() : MediaType.APPLICATION_OCTET_STREAM_VALUE; + String name = StringUtils.isNotBlank(bitstream.getName()) + ? bitstream.getName() : bitstream.getID().toString(); + String quotedEtag = "\"" + bitstream.getChecksum() + "\""; + long sizeBytes = bitstream.getSizeBytes(); + long lastModifiedMillis = item.getLastModified() != null ? item.getLastModified().getTime() : 0L; + + if (RequestMethod.HEAD.name().equals(request.getMethod())) { + // HEAD is always answered locally (never redirected to S3), so that HuggingFace clients can + // read the metadata headers without following a presigned URL. + HttpHeaders headers = buildCommonHeaders(sha, quotedEtag, sizeBytes, mimetype, lastModifiedMillis); + context.complete(); + return ResponseEntity.ok().headers(headers).build(); + } + + // We only log a download for requests without a Range header, because a client always sends a + // regular request first to check for Range support (mirrors BitstreamByHandleRestController). + if (StringUtils.isBlank(request.getHeader("Range"))) { + eventService.fireEvent( + new UsageEvent( + UsageEvent.Action.VIEW, + request, + context, + bitstream)); + + // Track the download in Matomo - only if the downloading has started (the condition is + // inside the method) + matomoBitstreamTracker.trackBitstreamDownload(context, request, bitstream, false); + } + + try { + boolean s3DirectDownload = configurationService.getBooleanProperty("s3.download.direct.enabled"); + boolean s3AssetstoreEnabled = configurationService.getBooleanProperty("assetstore.s3.enabled"); + if (s3DirectDownload && s3AssetstoreEnabled) { + // Download only files which are stored in the `ORIGINAL` bundle, because some specific + // files are not correctly downloaded and displayed in the UI when using presigned URLs. + boolean hasOriginalBundle = bitstream.getBundles().stream() + .anyMatch(bundle -> Constants.CONTENT_BUNDLE_NAME.equals(bundle.getName())); + if (hasOriginalBundle) { + HttpHeaders headers = + buildCommonHeaders(sha, quotedEtag, sizeBytes, mimetype, lastModifiedMillis); + // Close the DB connection before redirecting + context.complete(); + return redirectToS3DownloadUrl(headers, name, bitstream.getInternalId()); + } + } + + EPerson currentUser = context.getCurrentUser(); + org.dspace.app.rest.utils.BitstreamResource bitstreamResource = + new org.dspace.app.rest.utils.BitstreamResource(name, bitstream.getID(), + currentUser != null ? currentUser.getID() : null, + context.getSpecialGroupUuids(), false); + + HttpHeadersInitializer httpHeadersInitializer = new HttpHeadersInitializer() + .withBufferSize(BUFFER_SIZE) + .withFileName(name) + .withChecksum(quotedEtag) + .withLength(bitstreamResource.contentLength()) + .withMimetype(mimetype) + .withLastModified(lastModifiedMillis) + .withDisposition(HttpHeadersInitializer.CONTENT_DISPOSITION_ATTACHMENT) + .with(request) + .with(response); + + // We have all the data we need, close the connection to the database so that it doesn't + // stay open during download/streaming + context.complete(); + + if (httpHeadersInitializer.isValid()) { + HttpHeaders httpHeaders = httpHeadersInitializer.initialiseHeaders(); + httpHeaders.set(REPO_COMMIT_HEADER, sha); + httpHeaders.set(LINKED_ETAG_HEADER, quotedEtag); + httpHeaders.set(LINKED_SIZE_HEADER, String.valueOf(sizeBytes)); + return ResponseEntity.ok().headers(httpHeaders).body(bitstreamResource); + } + } catch (ClientAbortException ex) { + log.debug("Client aborted the request before the download was completed. " + + "Client is probably switching to a Range request.", ex); + } + return null; + } + + /** + * Build the metadata headers shared by every successful {@code resolve} response (HEAD, 302 redirect + * and streamed GET). + * + * @param sha the item's current sha + * @param quotedEtag the bitstream's MD5 checksum, wrapped in double quotes + * @param sizeBytes the bitstream size in bytes + * @param mimetype the bitstream MIME type + * @param lastModifiedMillis the item's last-modified timestamp, in epoch milliseconds + * @return the headers + */ + private HttpHeaders buildCommonHeaders(String sha, String quotedEtag, long sizeBytes, String mimetype, + long lastModifiedMillis) { + HttpHeaders headers = new HttpHeaders(); + headers.set(REPO_COMMIT_HEADER, sha); + headers.set(HttpHeaders.ETAG, quotedEtag); + headers.set(LINKED_ETAG_HEADER, quotedEtag); + headers.set(HttpHeaders.CONTENT_LENGTH, String.valueOf(sizeBytes)); + headers.set(LINKED_SIZE_HEADER, String.valueOf(sizeBytes)); + headers.set(HttpHeaders.ACCEPT_RANGES, "bytes"); + headers.set(HttpHeaders.CONTENT_TYPE, mimetype); + headers.set(HttpHeaders.LAST_MODIFIED, FastHttpDateFormat.formatDate(lastModifiedMillis)); + return headers; + } + + /** + * Extract the requested filename from the raw request URI of a {@code resolve} request: everything + * after the {@code /resolve/} marker and the following (revision) path segment, decoded once. This + * supports filenames containing slashes, spaces, parentheses and other URL-encoded characters. + * + * @param request the HTTP request + * @return the decoded filename, or {@code null} if the URI contains no filename part + */ + private String extractResolveFilename(HttpServletRequest request) { + String uri = request.getRequestURI(); + int markerIndex = uri.indexOf(RESOLVE_MARKER); + if (markerIndex < 0) { + return null; + } + String rest = uri.substring(markerIndex + RESOLVE_MARKER.length()); + int slashIndex = rest.indexOf('/'); + if (slashIndex < 0) { + return null; + } + String rawFilename = rest.substring(slashIndex + 1); + return UriUtils.decode(rawFilename, StandardCharsets.UTF_8); + } + + /** + * This method will handle the S3 direct download by generating a presigned URL for the bitstream and + * returning a redirect response to the client (mirrors {@code BitstreamRestController}). + * + * @param httpHeaders headers needed to form a proper response when returning the Bitstream/File + * @param bitName name of the bitstream + * @param bitInternalId internal id of the bitstream + * @return ResponseEntity with the location header set to the presigned URL + */ + private ResponseEntity redirectToS3DownloadUrl(HttpHeaders httpHeaders, String bitName, + String bitInternalId) { + try { + String bucket = configurationService.getProperty("assetstore.s3.bucketName", ""); + if (StringUtils.isBlank(bucket)) { + throw new InternalServerErrorException("S3 bucket name is not configured"); + } + + // Get the full path to the bitstream in the S3 bucket + String bitstreamPath = s3BitStoreService.getFullKey(bitInternalId); + if (StringUtils.isBlank(bitstreamPath)) { + throw new InternalServerErrorException("Failed to get bitstream path for internal ID: " + + bitInternalId); + } + + // Generate a presigned URL for the bitstream with a configurable expiration time + int expirationTime = configurationService.getIntProperty("s3.download.direct.expiration", 3600); + log.debug("Generating presigned URL with expiration time of {} seconds", expirationTime); + String presignedUrl = + s3DirectDownloadService.generatePresignedUrl(bucket, bitstreamPath, expirationTime, bitName); + + if (StringUtils.isBlank(presignedUrl)) { + throw new InternalServerErrorException("Failed to generate presigned URL for bitstream: " + + bitInternalId); + } + + // Set the Location header to the presigned URL - this will redirect the client to the S3 URL + httpHeaders.setLocation(URI.create(presignedUrl)); + return ResponseEntity.status(HttpStatus.FOUND).headers(httpHeaders).build(); + } catch (Exception e) { + throw new InternalServerErrorException("Error generating S3 presigned URL for bitstream: " + + bitInternalId, e); + } + } + + /** + * Common flow shared by the model-info, revision and tree endpoints: resolve the handle to an + * exposed Item, check READ authorization on it, and compute its current sha. + * + * @param context the DSpace context + * @param prefix the handle prefix + * @param suffix the handle suffix + * @return the resolved item together with its current sha + * @throws SQLException if a database error occurs while resolving the handle + * @throws HuggingFaceApiException if the handle does not resolve to an exposed item, or the + * current user is not authorized to read it + */ + private ResolvedRepository resolveAuthorizedRepository(Context context, String prefix, String suffix) + throws SQLException { + String handle = prefix + "/" + suffix; + + if (Objects.isNull(context)) { + log.error("Cannot obtain the context from the request."); + throw new HuggingFaceApiException(HttpStatus.INTERNAL_SERVER_ERROR, null, "Internal error."); + } + + Item item = huggingFaceService.resolveExposedItem(context, handle); + if (item == null) { + throw new HuggingFaceApiException(HttpStatus.NOT_FOUND, "RepoNotFound", "Repository not found"); + } + + try { + authorizeService.authorizeAction(context, item, Constants.READ); + } catch (AuthorizeException e) { + // TODO: phase 2 - Bearer token auth: map Authorization: Bearer to a CLARIN Personal Access + // Token and re-check authorization with the resolved EPerson before falling back to the + // anonymous 401/403 distinction below. + log.warn("Unauthorized access to HuggingFace repository '{}'.", handle); + HttpStatus status = context.getCurrentUser() == null ? HttpStatus.UNAUTHORIZED : HttpStatus.FORBIDDEN; + throw new HuggingFaceApiException(status, null, "You are not authorized to access this repository."); + } + + String sha = huggingFaceService.computeShaForItem(context, item); + return new ResolvedRepository(item, sha); + } + + /** + * Validate a requested revision against the resolved item's current sha. + * + * @param revision the requested revision + * @param currentSha the item's current sha + * @throws HuggingFaceApiException if the revision is not valid + */ + private void requireValidRevision(String revision, String currentSha) { + if (!ClarinHuggingFaceService.isValidRevision(revision, currentSha)) { + throw new HuggingFaceApiException(HttpStatus.NOT_FOUND, "RevisionNotFound", "Revision not found"); + } + } + + /** + * Simple holder for the result of {@link #resolveAuthorizedRepository(Context, String, String)}. + */ + private static final class ResolvedRepository { + + private final Item item; + + private final String sha; + + private ResolvedRepository(Item item, String sha) { + this.item = item; + this.sha = sha; + } + } + + /** + * Internal unchecked exception used to short-circuit the common resolve/authorize/validate-revision + * flow shared by the model-info, revision and tree endpoints, carrying the HTTP status, optional + * {@code X-Error-Code} header value and JSON error message to send back to the client. + */ + private static final class HuggingFaceApiException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final HttpStatus status; + + private final String errorCode; + + private final String message; + + private HuggingFaceApiException(HttpStatus status, String errorCode, String message) { + this.status = status; + this.errorCode = errorCode; + this.message = message; + } + + /** + * Build the JSON error {@link ResponseEntity} for this exception. + * + * @return the response entity + */ + private ResponseEntity toResponse() { + ResponseEntity.BodyBuilder builder = ResponseEntity.status(status); + if (errorCode != null) { + builder = builder.header(ERROR_CODE_HEADER, errorCode); + } + return builder.body(Collections.singletonMap("error", message)); + } + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java new file mode 100644 index 000000000000..3617e1c739fc --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java @@ -0,0 +1,498 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.authorize.AuthorizeException; +import org.dspace.content.Bitstream; +import org.dspace.content.Bundle; +import org.dspace.content.Collection; +import org.dspace.content.DSpaceObject; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.service.ItemService; +import org.dspace.core.Constants; +import org.dspace.core.Context; +import org.dspace.handle.service.HandleService; +import org.dspace.services.ConfigurationService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * Service backing the (future, Milestone 2) HuggingFace-Hub-compatible facade controller. + * + *

This service is responsible for:

+ *
    + *
  • deciding whether the facade is enabled at all ({@link #isEnabled()});
    • + *
  • deciding whether a given Item is allowed to be exposed through the facade + * ({@link #isExposed(Context, Item)});
    • + *
  • resolving a handle to an exposed Item ({@link #resolveExposedItem(Context, String)});
    • + *
  • computing a deterministic "sha" (revision) for an Item's ORIGINAL bitstreams + * ({@link #computeSha(String, long, Map)} / {@link #computeShaForItem(Context, Item)});
    • + *
  • validating a requested revision against the current sha ({@link #isValidRevision(String, String)});
    • + *
  • building the HuggingFace-Hub-shaped JSON model ({@link #toModelInfo(Context, Item, String)}) and + * file tree ({@link #toTreeEntries(Context, Item)}) for an Item.
    • + *
+ * + * @author DSpace at UFAL + */ +@Component +public class ClarinHuggingFaceService { + + private static final Logger log = LogManager.getLogger(ClarinHuggingFaceService.class); + + private static final String PROP_ENABLED = "hf.api.enabled"; + + private static final String PROP_FILTER_METADATA_FIELD = "hf.api.filter.metadata-field"; + + private static final String PROP_FILTER_VALUE = "hf.api.filter.value"; + + private static final String PROP_EXPOSED_COLLECTIONS = "hf.api.exposed.collections"; + + private static final String DEFAULT_FILTER_METADATA_FIELD = "dc.type"; + + private static final String MAIN_REVISION = "main"; + + private static final int DEFAULT_LIST_LIMIT = 20; + + private static final int MAX_LIST_LIMIT = 100; + + /** + * Guard so that the "no exposure filter configured" warning is only logged once per JVM lifetime, + * instead of once per checked Item. + */ + private static final AtomicBoolean NO_FILTER_CONFIGURED_WARNING_LOGGED = new AtomicBoolean(false); + + @Autowired + private ConfigurationService configurationService; + + @Autowired + private ItemService itemService; + + @Autowired + private HandleService handleService; + + /** + * Master switch for the HuggingFace-Hub-compatible facade. + * + * @return {@code true} if the facade is enabled via {@code hf.api.enabled}, {@code false} otherwise + */ + public boolean isEnabled() { + return configurationService.getBooleanProperty(PROP_ENABLED, false); + } + + /** + * Fail-closed exposure filter deciding whether a given Item may be exposed through the HuggingFace facade. + * + *

An Item is exposed iff:

+ *
    + *
  • the item is non-null, archived and not withdrawn, AND
    • + *
  • either (a) the configured metadata field ({@code hf.api.filter.metadata-field}) has a value on + * the item equal to the configured filter value ({@code hf.api.filter.value}) -- only evaluated when + * both properties are non-blank -- or (b) the item's owning collection handle is listed in + * {@code hf.api.exposed.collections}.
    • + *
+ * + *

If neither the metadata filter nor the collection allow-list is configured, a warning is logged + * (once) and the method fails closed, returning {@code false}.

+ * + * @param context the DSpace context + * @param item the item to check + * @return {@code true} if the item may be exposed through the HuggingFace facade + */ + public boolean isExposed(Context context, Item item) { + if (item == null || item.isWithdrawn() || !item.isArchived()) { + return false; + } + + String filterMetadataField = configurationService.getProperty(PROP_FILTER_METADATA_FIELD, + DEFAULT_FILTER_METADATA_FIELD); + String filterValue = configurationService.getProperty(PROP_FILTER_VALUE); + boolean filterConfigured = StringUtils.isNotBlank(filterMetadataField) && StringUtils.isNotBlank(filterValue); + + String[] exposedCollections = configurationService.getArrayProperty(PROP_EXPOSED_COLLECTIONS); + boolean collectionsConfigured = exposedCollections != null && exposedCollections.length > 0; + + if (!filterConfigured && !collectionsConfigured) { + if (NO_FILTER_CONFIGURED_WARNING_LOGGED.compareAndSet(false, true)) { + log.warn("Neither '{}'/'{}' nor '{}' is configured; the HuggingFace facade will not expose " + + "any item.", PROP_FILTER_METADATA_FIELD, PROP_FILTER_VALUE, + PROP_EXPOSED_COLLECTIONS); + } + return false; + } + + if (filterConfigured) { + List values = itemService.getMetadataByMetadataString(item, filterMetadataField); + for (MetadataValue value : values) { + if (StringUtils.equals(filterValue, value.getValue())) { + return true; + } + } + } + + if (collectionsConfigured) { + Collection owningCollection = item.getOwningCollection(); + String ownerHandle = owningCollection != null ? owningCollection.getHandle() : null; + if (ownerHandle != null) { + for (String exposedHandle : exposedCollections) { + if (StringUtils.equals(ownerHandle, exposedHandle)) { + return true; + } + } + } + } + + return false; + } + + /** + * Resolve a handle to an Item that is allowed to be exposed through the HuggingFace facade. + * + * @param context the DSpace context + * @param handle the handle to resolve (e.g. {@code "11234/1-5814"}) + * @return the exposed Item, or {@code null} if the handle does not resolve to an Item, or the resolved + * Item is not exposed + * @throws SQLException if a database error occurs while resolving the handle + */ + public Item resolveExposedItem(Context context, String handle) throws SQLException { + DSpaceObject dso = handleService.resolveToObject(context, handle); + if (!(dso instanceof Item)) { + return null; + } + Item item = (Item) dso; + return isExposed(context, item) ? item : null; + } + + /** + * Find items exposed through the HuggingFace facade, optionally restricted to items whose title + * contains {@code search} (case-insensitively), and capped at {@code limit} results. + * + *

Candidates are collected -- deliberately without any Solr dependence -- from up to two sources, + * which are merged and deduplicated by item UUID:

+ *
    + *
  • every item of every collection listed in {@code hf.api.exposed.collections}, resolved via + * {@link HandleService} and iterated with {@link ItemService#findByCollection(Context, Collection)}; + * and
    • + *
  • the items whose configured filter metadata field ({@code hf.api.filter.metadata-field}) has + * the configured filter value ({@code hf.api.filter.value}), via + * {@link ItemService#findArchivedByMetadataField(Context, String, String, String, String)}.
    • + *
+ * + *

Every candidate is (re-)checked against {@link #isExposed(Context, Item)}, fail-closed, which also + * filters out withdrawn/non-archived items that may still be reachable through the collection path.

+ * + * @param context the DSpace context + * @param search an optional, case-insensitive substring to match against the item title (or name, if + * the item has no title); {@code null} or blank to not filter by title + * @param limit the maximum number of items to return; a non-positive value defaults to 20, and any + * value is capped at 100 + * @return the list of exposed items matching {@code search}, capped at the effective limit; empty if + * neither exposure mechanism is configured, or no item is exposed and matches + * @throws SQLException if a database error occurs + */ + public List findExposedItems(Context context, String search, int limit) throws SQLException { + int effectiveLimit = limit <= 0 ? DEFAULT_LIST_LIMIT : Math.min(limit, MAX_LIST_LIMIT); + + LinkedHashMap candidates = new LinkedHashMap<>(); + + String[] exposedCollections = configurationService.getArrayProperty(PROP_EXPOSED_COLLECTIONS); + if (exposedCollections != null) { + for (String handle : exposedCollections) { + DSpaceObject dso = handleService.resolveToObject(context, handle); + if (dso instanceof Collection) { + Iterator items = itemService.findByCollection(context, (Collection) dso); + while (items.hasNext()) { + Item item = items.next(); + candidates.putIfAbsent(item.getID(), item); + } + } + } + } + + String filterMetadataField = configurationService.getProperty(PROP_FILTER_METADATA_FIELD, + DEFAULT_FILTER_METADATA_FIELD); + String filterValue = configurationService.getProperty(PROP_FILTER_VALUE); + if (StringUtils.isNotBlank(filterMetadataField) && StringUtils.isNotBlank(filterValue)) { + String[] fieldParts = filterMetadataField.split("\\."); + String schema = fieldParts.length > 0 ? fieldParts[0] : null; + String element = fieldParts.length > 1 ? fieldParts[1] : null; + String qualifier = fieldParts.length > 2 ? fieldParts[2] : null; + try { + Iterator items = + itemService.findArchivedByMetadataField(context, schema, element, qualifier, filterValue); + while (items.hasNext()) { + Item item = items.next(); + candidates.putIfAbsent(item.getID(), item); + } + } catch (AuthorizeException e) { + log.warn("Unexpected AuthorizeException while listing HuggingFace-exposed items by metadata " + + "filter '{}' = '{}'.", filterMetadataField, filterValue, e); + } + } + + String searchLower = StringUtils.isNotBlank(search) ? search.toLowerCase(Locale.ROOT) : null; + + List result = new ArrayList<>(); + for (Item item : candidates.values()) { + if (result.size() >= effectiveLimit) { + break; + } + if (!isExposed(context, item)) { + continue; + } + if (searchLower != null && !matchesSearch(item, searchLower)) { + continue; + } + result.add(item); + } + + return result; + } + + /** + * Check whether an Item's title (or name, if it has no title) contains the given (already + * lower-cased) search string. + * + * @param item the item + * @param searchLower the search string, already lower-cased + * @return {@code true} if the item's title/name contains {@code searchLower}, case-insensitively + */ + private boolean matchesSearch(Item item, String searchLower) { + String title = itemService.getMetadataFirstValue(item, "dc", "title", null, Item.ANY); + if (StringUtils.isBlank(title)) { + title = item.getName(); + } + return title != null && title.toLowerCase(Locale.ROOT).contains(searchLower); + } + + /** + * Compute a deterministic sha (revision id) for an Item, given its last-modified timestamp and the + * filename-to-md5 map of its ORIGINAL bitstreams. + * + *

The result is a 40-character lowercase hex string (SHA-1), deterministic across JVM runs and + * insensitive to the iteration order of {@code filenameToMd5}.

+ * + * @param itemUuid the item's UUID, as a string + * @param lastModifiedMillis the item's last-modified timestamp, in epoch milliseconds + * @param filenameToMd5 map of ORIGINAL bitstream filename to its MD5 checksum + * @return the computed sha, a 40-character lowercase hex string + */ + public static String computeSha(String itemUuid, long lastModifiedMillis, Map filenameToMd5) { + List pairs = new ArrayList<>(); + for (Map.Entry entry : filenameToMd5.entrySet()) { + pairs.add(entry.getKey() + ":" + entry.getValue()); + } + Collections.sort(pairs); + String joined = String.join("\n", pairs); + String input = itemUuid + "\n" + lastModifiedMillis + "\n" + joined; + return DigestUtils.sha1Hex(input); + } + + /** + * Check whether a requested revision string is valid, given the current sha of an Item. + * + * @param revision the requested revision (path parameter of the facade endpoints) + * @param currentSha the current sha of the Item, as computed by {@link #computeSha(String, long, Map)} + * @return {@code true} if {@code revision} equals {@code "main"} or equals {@code currentSha} + * (case-insensitively); {@code false} if {@code revision} is null or blank + */ + public static boolean isValidRevision(String revision, String currentSha) { + if (StringUtils.isBlank(revision)) { + return false; + } + return MAIN_REVISION.equals(revision) || revision.equalsIgnoreCase(currentSha); + } + + /** + * Compute the current sha of an Item, based on its ORIGINAL bitstreams and last-modified timestamp. + * + * @param context the DSpace context + * @param item the item + * @return the computed sha, a 40-character lowercase hex string + */ + public String computeShaForItem(Context context, Item item) { + Map originalFiles = getOriginalFiles(item); + Map filenameToMd5 = new LinkedHashMap<>(); + for (Map.Entry entry : originalFiles.entrySet()) { + filenameToMd5.put(entry.getKey(), entry.getValue().getChecksum()); + } + long lastModifiedMillis = item.getLastModified() != null ? item.getLastModified().getTime() : 0L; + String itemUuid = item.getID().toString(); + return computeSha(itemUuid, lastModifiedMillis, filenameToMd5); + } + + /** + * Collect the filename-to-bitstream map of all bitstreams in the Item's ORIGINAL bundles, iterating + * bundles then bitstreams in order. If several bitstreams share the same filename, the first one + * encountered wins. + * + * @param item the item + * @return a filename-to-bitstream map, preserving encounter order; empty if the item has no ORIGINAL + * bundle or the bundle(s) are empty + */ + public LinkedHashMap getOriginalFiles(Item item) { + LinkedHashMap result = new LinkedHashMap<>(); + List bundles = item.getBundles(Constants.CONTENT_BUNDLE_NAME); + for (Bundle bundle : bundles) { + for (Bitstream bitstream : bundle.getBitstreams()) { + result.putIfAbsent(bitstream.getName(), bitstream); + } + } + return result; + } + + /** + * Build the HuggingFace-Hub-shaped model info for an Item. + * + * @param context the DSpace context + * @param item the item + * @param sha the item's current sha, as computed by {@link #computeShaForItem(Context, Item)} + * @return the model info + */ + public HuggingFaceModelInfo toModelInfo(Context context, Item item, String sha) { + HuggingFaceModelInfo modelInfo = new HuggingFaceModelInfo(); + + String handle = item.getHandle(); + modelInfo.setId(handle); + modelInfo.setModelId(handle); + modelInfo.setAuthor(handleAuthor(handle)); + modelInfo.setSha(sha); + + LinkedHashSet tags = new LinkedHashSet<>(); + addMetadataValues(tags, itemService.getMetadataByMetadataString(item, "dc.subject")); + List languages = metadataValues(itemService.getMetadataByMetadataString(item, "dc.language.iso")); + tags.addAll(languages); + addMetadataValues(tags, itemService.getMetadataByMetadataString(item, "dc.type")); + modelInfo.setTags(new ArrayList<>(tags)); + + Map cardData = new LinkedHashMap<>(); + List rights = itemService.getMetadataByMetadataString(item, "dc.rights"); + if (!rights.isEmpty()) { + cardData.put("license", rights.get(0).getValue()); + } + if (!languages.isEmpty()) { + cardData.put("language", languages); + } + cardData.put("tags", new ArrayList<>(tags)); + modelInfo.setCardData(cardData); + + long lastModifiedMillis = item.getLastModified() != null ? item.getLastModified().getTime() : 0L; + String lastModified = Instant.ofEpochMilli(lastModifiedMillis).toString(); + modelInfo.setLastModified(lastModified); + modelInfo.setCreatedAt(resolveCreatedAt(item, lastModified)); + + List siblings = new ArrayList<>(); + for (String filename : getOriginalFiles(item).keySet()) { + siblings.add(new HuggingFaceSibling(filename)); + } + modelInfo.setSiblings(siblings); + + modelInfo.setPrivate(false); + modelInfo.setGated(false); + modelInfo.setDisabled(false); + modelInfo.setDownloads(0L); + modelInfo.setLikes(0L); + + return modelInfo; + } + + /** + * Build the HuggingFace-Hub-shaped file tree for an Item, one entry per ORIGINAL bitstream. + * + * @param context the DSpace context + * @param item the item + * @return the list of tree entries; empty if the item has no ORIGINAL bitstreams + */ + public List toTreeEntries(Context context, Item item) { + List entries = new ArrayList<>(); + for (Map.Entry entry : getOriginalFiles(item).entrySet()) { + HuggingFaceTreeEntry treeEntry = new HuggingFaceTreeEntry(); + treeEntry.setType("file"); + treeEntry.setPath(entry.getKey()); + treeEntry.setSize(entry.getValue().getSizeBytes()); + treeEntry.setOid(entry.getValue().getChecksum()); + entries.add(treeEntry); + } + return entries; + } + + /** + * Extract the author (handle prefix) from a handle. + * + * @param handle the handle, e.g. {@code "11234/1-5814"} + * @return the handle prefix, e.g. {@code "11234"}, or the handle itself if it does not contain a slash + */ + private String handleAuthor(String handle) { + if (handle == null) { + return null; + } + int slashIndex = handle.indexOf('/'); + return slashIndex >= 0 ? handle.substring(0, slashIndex) : handle; + } + + /** + * Resolve the "createdAt" timestamp of an Item from its {@code dc.date.accessioned} metadata, falling + * back to the given last-modified timestamp if the metadata is missing or unparseable. + * + * @param item the item + * @param lastModified the pre-computed, ISO-8601 last-modified timestamp to fall back on + * @return the ISO-8601 createdAt timestamp + */ + private String resolveCreatedAt(Item item, String lastModified) { + List accessioned = itemService.getMetadataByMetadataString(item, "dc.date.accessioned"); + if (accessioned.isEmpty()) { + return lastModified; + } + try { + return Instant.parse(accessioned.get(0).getValue()).toString(); + } catch (RuntimeException e) { + return lastModified; + } + } + + /** + * Extract the {@link MetadataValue#getValue()} of each entry into a new list. + * + * @param values the metadata values + * @return the extracted string values, in the same order + */ + private List metadataValues(List values) { + List result = new ArrayList<>(); + addMetadataValues(result, values); + return result; + } + + /** + * Append the {@link MetadataValue#getValue()} of each entry to the given collection. + * + * @param target the collection to append to + * @param values the metadata values + */ + private void addMetadataValues(java.util.Collection target, List values) { + for (MetadataValue value : values) { + target.add(value.getValue()); + } + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceModelInfo.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceModelInfo.java new file mode 100644 index 000000000000..f4b02c5e8ed2 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceModelInfo.java @@ -0,0 +1,312 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Plain Jackson POJO modeling the JSON shape expected by huggingface_hub's + * {@code model_info} response (as consumed by the {@code huggingface_hub} Python + * client library). This class is intentionally decoupled from the DSpace REST + * HAL/RestRepository machinery: it is a simple data holder that is serialized + * directly to JSON by the (future) HuggingFace-facade controller. + * + * @author DSpace at UFAL + */ +public class HuggingFaceModelInfo { + + private String id; + + private String modelId; + + private String author; + + private String sha; + + private String lastModified; + + private String createdAt; + + @JsonProperty("private") + private boolean isPrivate; + + private boolean gated; + + private boolean disabled; + + private long downloads; + + private long likes; + + private List tags; + + private Map cardData; + + private List siblings; + + /** + * Default no-arg constructor required for Jackson (de)serialization. + */ + public HuggingFaceModelInfo() { + } + + /** + * Get the repository id (in our case, the Item handle). + * + * @return the repository id + */ + public String getId() { + return id; + } + + /** + * Set the repository id. + * + * @param id the repository id + */ + public void setId(String id) { + this.id = id; + } + + /** + * Get the model id (in our case, the same value as {@link #getId()}). + * + * @return the model id + */ + public String getModelId() { + return modelId; + } + + /** + * Set the model id. + * + * @param modelId the model id + */ + public void setModelId(String modelId) { + this.modelId = modelId; + } + + /** + * Get the author (in our case, the handle prefix). + * + * @return the author + */ + public String getAuthor() { + return author; + } + + /** + * Set the author. + * + * @param author the author + */ + public void setAuthor(String author) { + this.author = author; + } + + /** + * Get the sha (revision) of the repository content. + * + * @return the sha + */ + public String getSha() { + return sha; + } + + /** + * Set the sha (revision) of the repository content. + * + * @param sha the sha + */ + public void setSha(String sha) { + this.sha = sha; + } + + /** + * Get the last modification timestamp, as an ISO-8601 string. + * + * @return the last modification timestamp + */ + public String getLastModified() { + return lastModified; + } + + /** + * Set the last modification timestamp. + * + * @param lastModified the last modification timestamp, as an ISO-8601 string + */ + public void setLastModified(String lastModified) { + this.lastModified = lastModified; + } + + /** + * Get the creation timestamp, as an ISO-8601 string. + * + * @return the creation timestamp + */ + public String getCreatedAt() { + return createdAt; + } + + /** + * Set the creation timestamp. + * + * @param createdAt the creation timestamp, as an ISO-8601 string + */ + public void setCreatedAt(String createdAt) { + this.createdAt = createdAt; + } + + /** + * Whether the repository is private. Always {@code false} for items exposed by this facade. + * + * @return whether the repository is private + */ + public boolean isPrivate() { + return isPrivate; + } + + /** + * Set whether the repository is private. + * + * @param isPrivate whether the repository is private + */ + public void setPrivate(boolean isPrivate) { + this.isPrivate = isPrivate; + } + + /** + * Whether the repository is gated. Always {@code false} for items exposed by this facade. + * + * @return whether the repository is gated + */ + public boolean isGated() { + return gated; + } + + /** + * Set whether the repository is gated. + * + * @param gated whether the repository is gated + */ + public void setGated(boolean gated) { + this.gated = gated; + } + + /** + * Whether the repository is disabled. Always {@code false} for items exposed by this facade. + * + * @return whether the repository is disabled + */ + public boolean isDisabled() { + return disabled; + } + + /** + * Set whether the repository is disabled. + * + * @param disabled whether the repository is disabled + */ + public void setDisabled(boolean disabled) { + this.disabled = disabled; + } + + /** + * Get the number of downloads. Always {@code 0} for items exposed by this facade. + * + * @return the number of downloads + */ + public long getDownloads() { + return downloads; + } + + /** + * Set the number of downloads. + * + * @param downloads the number of downloads + */ + public void setDownloads(long downloads) { + this.downloads = downloads; + } + + /** + * Get the number of likes. Always {@code 0} for items exposed by this facade. + * + * @return the number of likes + */ + public long getLikes() { + return likes; + } + + /** + * Set the number of likes. + * + * @param likes the number of likes + */ + public void setLikes(long likes) { + this.likes = likes; + } + + /** + * Get the tags associated with this repository. + * + * @return the tags + */ + public List getTags() { + return tags; + } + + /** + * Set the tags associated with this repository. + * + * @param tags the tags + */ + public void setTags(List tags) { + this.tags = tags; + } + + /** + * Get the card data (metadata block normally found in the model card's YAML front matter). + * + * @return the card data + */ + public Map getCardData() { + return cardData; + } + + /** + * Set the card data. + * + * @param cardData the card data + */ + public void setCardData(Map cardData) { + this.cardData = cardData; + } + + /** + * Get the list of files (siblings) belonging to this repository. + * + * @return the siblings + */ + public List getSiblings() { + return siblings; + } + + /** + * Set the list of files (siblings) belonging to this repository. + * + * @param siblings the siblings + */ + public void setSiblings(List siblings) { + this.siblings = siblings; + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceSibling.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceSibling.java new file mode 100644 index 000000000000..1ab614cad756 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceSibling.java @@ -0,0 +1,54 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +/** + * Plain Jackson POJO representing a single entry of the {@code siblings} array + * returned by the HuggingFace Hub {@code model_info} API. Each sibling corresponds + * to a single file that belongs to the "model repository" (in our case, an ORIGINAL + * bitstream of a DSpace Item). + * + * @author DSpace at UFAL + */ +public class HuggingFaceSibling { + + private String rfilename; + + /** + * Default no-arg constructor required for Jackson (de)serialization. + */ + public HuggingFaceSibling() { + } + + /** + * Construct a sibling entry for the given filename. + * + * @param rfilename the repository-relative filename + */ + public HuggingFaceSibling(String rfilename) { + this.rfilename = rfilename; + } + + /** + * Get the repository-relative filename. + * + * @return the filename + */ + public String getRfilename() { + return rfilename; + } + + /** + * Set the repository-relative filename. + * + * @param rfilename the filename + */ + public void setRfilename(String rfilename) { + this.rfilename = rfilename; + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceTreeEntry.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceTreeEntry.java new file mode 100644 index 000000000000..e7e79d31ac50 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/HuggingFaceTreeEntry.java @@ -0,0 +1,105 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +/** + * Plain Jackson POJO representing a single entry returned by the HuggingFace Hub + * "tree" endpoint (e.g. {@code GET /api/models/{repo_id}/tree/{revision}}). Each entry + * describes one file present in the repository (in our case, an ORIGINAL bitstream of + * a DSpace Item). + * + * @author DSpace at UFAL + */ +public class HuggingFaceTreeEntry { + + private String type; + + private String path; + + private long size; + + private String oid; + + /** + * Default no-arg constructor required for Jackson (de)serialization. + */ + public HuggingFaceTreeEntry() { + } + + /** + * Get the entry type, e.g. {@code "file"}. + * + * @return the entry type + */ + public String getType() { + return type; + } + + /** + * Set the entry type. + * + * @param type the entry type + */ + public void setType(String type) { + this.type = type; + } + + /** + * Get the repository-relative path of the file. + * + * @return the file path + */ + public String getPath() { + return path; + } + + /** + * Set the repository-relative path of the file. + * + * @param path the file path + */ + public void setPath(String path) { + this.path = path; + } + + /** + * Get the file size in bytes. + * + * @return the file size in bytes + */ + public long getSize() { + return size; + } + + /** + * Set the file size in bytes. + * + * @param size the file size in bytes + */ + public void setSize(long size) { + this.size = size; + } + + /** + * Get the object id (checksum) of the file, as reported by the git/LFS-like tree API. + * + * @return the object id + */ + public String getOid() { + return oid; + } + + /** + * Set the object id (checksum) of the file. + * + * @param oid the object id + */ + public void setOid(String oid) { + this.oid = oid; + } +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java new file mode 100644 index 000000000000..162d0ab52365 --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java @@ -0,0 +1,404 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.matchesPattern; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertEquals; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.head; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.io.InputStream; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.codec.CharEncoding; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.IOUtils; +import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.builder.BitstreamBuilder; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.builder.ResourcePolicyBuilder; +import org.dspace.content.Bitstream; +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.content.service.ItemService; +import org.dspace.core.Constants; +import org.dspace.services.ConfigurationService; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.web.servlet.MvcResult; + +/** + * Integration tests for {@link ClarinHuggingFaceController}. + */ +public class ClarinHuggingFaceControllerIT extends AbstractControllerIntegrationTest { + + private static final String ENDPOINT_BASE = "/api/hf/api/models"; + + private static final String ZERO_REVISION = "0000000000000000000000000000000000000000"; + + private static final String MODEL_BIN_CONTENT = "hello model"; + + private static final String CONFIG_JSON_CONTENT = "config content"; + + @Autowired + private ConfigurationService configurationService; + + @Autowired + private AuthorizeService authorizeService; + + @Autowired + private ItemService itemService; + + private Item exposedItem; + + private Item nonExposedItem; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + + configurationService.setProperty("hf.api.enabled", true); + configurationService.setProperty("hf.api.filter.metadata-field", "dc.type"); + configurationService.setProperty("hf.api.filter.value", "machineLearningModel"); + + context.turnOffAuthorisationSystem(); + + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection collection = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + + exposedItem = ItemBuilder.createItem(context, collection) + .withTitle("A Machine Learning Model") + .withType("machineLearningModel") + .withSubject("nlp") + .withSubject("classification") + .build(); + try (InputStream is = IOUtils.toInputStream(MODEL_BIN_CONTENT, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, exposedItem, is) + .withName("model.bin") + .withMimeType("application/octet-stream") + .build(); + } + try (InputStream is = IOUtils.toInputStream(CONFIG_JSON_CONTENT, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, exposedItem, is) + .withName("config.json") + .withMimeType("application/json") + .build(); + } + + nonExposedItem = ItemBuilder.createItem(context, collection) + .withTitle("A Plain Item") + .build(); + + context.restoreAuthSystemState(); + } + + @Test + public void modelInfoReturnsExpectedShape() throws Exception { + String[] handleParts = exposedItem.getHandle().split("/"); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1])) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.id", is(exposedItem.getHandle()))) + .andExpect(jsonPath("$.sha", matchesPattern("^[0-9a-f]{40}$"))) + .andExpect(jsonPath("$.private", is(false))) + .andExpect(jsonPath("$.gated", is(false))) + .andExpect(jsonPath("$.downloads", is(0))) + .andExpect(jsonPath("$.siblings", hasSize(2))) + .andExpect(jsonPath("$.siblings[*].rfilename", hasItem("model.bin"))) + .andExpect(jsonPath("$.siblings[*].rfilename", hasItem("config.json"))) + .andExpect(jsonPath("$.tags", hasItem("nlp"))) + .andExpect(jsonPath("$.tags", hasItem("classification"))) + .andExpect(jsonPath("$.tags", hasItem("machineLearningModel"))); + } + + @Test + public void revisionMainIsValidAndZeroRevisionIsNotFound() throws Exception { + String[] handleParts = exposedItem.getHandle().split("/"); + String base = ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1]; + + getClient().perform(get(base + "/revision/main")) + .andExpect(status().isOk()); + + getClient().perform(get(base + "/revision/" + ZERO_REVISION)) + .andExpect(status().isNotFound()) + .andExpect(header().string("X-Error-Code", "RevisionNotFound")); + } + + @Test + public void treeMainReturnsFileEntriesWithCorrectSizeAndOid() throws Exception { + String[] handleParts = exposedItem.getHandle().split("/"); + + MvcResult result = getClient() + .perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/tree/main")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(2))) + .andExpect(jsonPath("$[*].type", everyItem(is("file")))) + .andReturn(); + + JsonNode entries = new ObjectMapper().readTree(result.getResponse().getContentAsString()); + Map byPath = new HashMap<>(); + for (JsonNode entry : entries) { + byPath.put(entry.get("path").asText(), entry); + } + + assertEquals(MODEL_BIN_CONTENT.length(), byPath.get("model.bin").get("size").asLong()); + assertEquals(DigestUtils.md5Hex(MODEL_BIN_CONTENT), byPath.get("model.bin").get("oid").asText()); + assertEquals(CONFIG_JSON_CONTENT.length(), byPath.get("config.json").get("size").asLong()); + assertEquals(DigestUtils.md5Hex(CONFIG_JSON_CONTENT), byPath.get("config.json").get("oid").asText()); + } + + @Test + public void treeAcceptsCurrentSha() throws Exception { + String[] handleParts = exposedItem.getHandle().split("/"); + String base = ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1]; + + MvcResult result = getClient().perform(get(base)) + .andExpect(status().isOk()) + .andReturn(); + String content = result.getResponse().getContentAsString(); + JsonNode json = new ObjectMapper().readTree(content); + String sha = json.get("sha").asText(); + + getClient().perform(get(base + "/tree/" + sha)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(2))); + } + + @Test + public void nonExposedItemIsNotFound() throws Exception { + String[] handleParts = nonExposedItem.getHandle().split("/"); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1])) + .andExpect(status().isNotFound()) + .andExpect(header().string("X-Error-Code", "RepoNotFound")); + } + + @Test + public void listModelsReturnsOnlyExposedItems() throws Exception { + context.turnOffAuthorisationSystem(); + Collection collection = exposedItem.getOwningCollection(); + Item secondExposedItem = ItemBuilder.createItem(context, collection) + .withTitle("Another Machine Learning Model") + .withType("machineLearningModel") + .build(); + context.restoreAuthSystemState(); + + getClient().perform(get(ENDPOINT_BASE)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$[*].id", hasItem(exposedItem.getHandle()))) + .andExpect(jsonPath("$[*].id", hasItem(secondExposedItem.getHandle()))) + .andExpect(jsonPath("$[*].id", not(hasItem(nonExposedItem.getHandle())))); + } + + @Test + public void listModelsSupportsSearchAndLimit() throws Exception { + context.turnOffAuthorisationSystem(); + Collection collection = exposedItem.getOwningCollection(); + ItemBuilder.createItem(context, collection) + .withTitle("Second Model") + .withType("machineLearningModel") + .build(); + context.restoreAuthSystemState(); + + getClient().perform(get(ENDPOINT_BASE).param("search", "Machine Learning")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(1))) + .andExpect(jsonPath("$[0].id", is(exposedItem.getHandle()))); + + getClient().perform(get(ENDPOINT_BASE).param("limit", "1")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(1))); + } + + @Test + public void listModelsDisabledFacadeReturnsNotFound() throws Exception { + configurationService.setProperty("hf.api.enabled", false); + + getClient().perform(get(ENDPOINT_BASE)) + .andExpect(status().isNotFound()); + } + + @Test + public void disabledFacadeReturnsNotFound() throws Exception { + configurationService.setProperty("hf.api.enabled", false); + String[] handleParts = exposedItem.getHandle().split("/"); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1])) + .andExpect(status().isNotFound()); + } + + @Test + public void withdrawnItemIsNotFound() throws Exception { + String[] handleParts = exposedItem.getHandle().split("/"); + + context.turnOffAuthorisationSystem(); + itemService.withdraw(context, exposedItem); + context.restoreAuthSystemState(); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1])) + .andExpect(status().isNotFound()) + .andExpect(header().string("X-Error-Code", "RepoNotFound")); + } + + @Test + public void whoAmIAlwaysReturnsUnauthorized() throws Exception { + getClient().perform(get("/api/hf/api/whoami-v2")) + .andExpect(status().isUnauthorized()); + } + + @Test + public void resolveHeadReturnsMetadataHeaders() throws Exception { + String handle = exposedItem.getHandle(); + String sha = fetchModelSha(handle); + + getClient().perform(head("/api/hf/" + handle + "/resolve/main/model.bin")) + .andExpect(status().isOk()) + .andExpect(header().string("X-Repo-Commit", sha)) + .andExpect(header().string("ETag", "\"" + DigestUtils.md5Hex(MODEL_BIN_CONTENT) + "\"")) + .andExpect(header().longValue("Content-Length", MODEL_BIN_CONTENT.length())) + .andExpect(header().string("Accept-Ranges", "bytes")) + .andExpect(header().exists("X-Linked-Etag")); + } + + @Test + public void resolveGetStreamsFileContent() throws Exception { + String handle = exposedItem.getHandle(); + + getClient().perform(get("/api/hf/" + handle + "/resolve/main/model.bin")) + .andExpect(status().isOk()) + .andExpect(header().exists("X-Repo-Commit")) + .andExpect(content().bytes(MODEL_BIN_CONTENT.getBytes())); + } + + @Test + public void resolveGetSupportsRangeRequests() throws Exception { + String handle = exposedItem.getHandle(); + + getClient().perform(get("/api/hf/" + handle + "/resolve/main/model.bin") + .header("Range", "bytes=1-3")) + .andExpect(status().is(206)) + // The Content-Length must match the requested range + .andExpect(header().longValue("Content-Length", 3)) + // The server should indicate we support Range requests + .andExpect(header().string("Accept-Ranges", "bytes")) + // The ETag has to be based on the checksum + .andExpect(header().string("ETag", "\"" + DigestUtils.md5Hex(MODEL_BIN_CONTENT) + "\"")) + // The response should give us details about the range + .andExpect(header().string("Content-Range", "bytes 1-3/" + MODEL_BIN_CONTENT.length())) + // We only expect the bytes 1, 2 and 3 + .andExpect(content().bytes(MODEL_BIN_CONTENT.substring(1, 4).getBytes())); + } + + @Test + public void resolveAcceptsCurrentShaAndRejectsUnknownRevision() throws Exception { + String handle = exposedItem.getHandle(); + String sha = fetchModelSha(handle); + + getClient().perform(get("/api/hf/" + handle + "/resolve/" + sha + "/model.bin")) + .andExpect(status().isOk()) + .andExpect(content().bytes(MODEL_BIN_CONTENT.getBytes())); + + getClient().perform(get("/api/hf/" + handle + "/resolve/" + ZERO_REVISION + "/model.bin")) + .andExpect(status().isNotFound()) + .andExpect(header().string("X-Error-Code", "RevisionNotFound")); + } + + @Test + public void resolveRestrictedBitstreamReturnsUnauthorizedOrForbidden() throws Exception { + String handle = exposedItem.getHandle(); + + context.turnOffAuthorisationSystem(); + Bitstream restricted; + try (InputStream is = IOUtils.toInputStream("secret content", CharEncoding.UTF_8)) { + restricted = BitstreamBuilder.createBitstream(context, exposedItem, is) + .withName("secret.bin") + .withMimeType("application/octet-stream") + .build(); + } + // Remove all read policies from the bitstream and add a read policy only for admin + authorizeService.removeAllPolicies(context, restricted); + ResourcePolicyBuilder.createResourcePolicy(context, admin, null) + .withDspaceObject(restricted) + .withAction(Constants.READ) + .build(); + context.restoreAuthSystemState(); + + // Anonymous user should get 401 + getClient().perform(get("/api/hf/" + handle + "/resolve/main/secret.bin")) + .andExpect(status().isUnauthorized()); + + // Authenticated non-admin user should get 403 + String token = getAuthToken(eperson.getEmail(), password); + getClient(token).perform(get("/api/hf/" + handle + "/resolve/main/secret.bin")) + .andExpect(status().isForbidden()); + } + + @Test + public void resolveMissingFileReturnsEntryNotFound() throws Exception { + String handle = exposedItem.getHandle(); + + getClient().perform(get("/api/hf/" + handle + "/resolve/main/nope.bin")) + .andExpect(status().isNotFound()) + .andExpect(header().string("X-Error-Code", "EntryNotFound")); + } + + @Test + public void resolveSupportsUrlEncodedFilenames() throws Exception { + String handle = exposedItem.getHandle(); + String encodedFileContent = "encoded filename content"; + + context.turnOffAuthorisationSystem(); + try (InputStream is = IOUtils.toInputStream(encodedFileContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, exposedItem, is) + .withName("my model (v2).bin") + .withMimeType("application/octet-stream") + .build(); + } + context.restoreAuthSystemState(); + + getClient().perform(get(URI.create("/api/hf/" + handle + "/resolve/main/my%20model%20%28v2%29.bin"))) + .andExpect(status().isOk()) + .andExpect(content().bytes(encodedFileContent.getBytes())); + } + + /** + * Fetch the current sha of a model repository from its model-info JSON. + * + * @param handle the item handle + * @return the sha reported by the model-info endpoint + */ + private String fetchModelSha(String handle) throws Exception { + MvcResult result = getClient().perform(get(ENDPOINT_BASE + "/" + handle)) + .andExpect(status().isOk()) + .andReturn(); + JsonNode json = new ObjectMapper().readTree(result.getResponse().getContentAsString()); + return json.get("sha").asText(); + } +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/hf/ClarinHuggingFaceServiceTest.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/hf/ClarinHuggingFaceServiceTest.java new file mode 100644 index 000000000000..3f59d33eaecd --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/hf/ClarinHuggingFaceServiceTest.java @@ -0,0 +1,169 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.hf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.Test; + +/** + * Plain JUnit 4 unit tests (no Spring context) for the pure static helper methods of + * {@link ClarinHuggingFaceService}: {@link ClarinHuggingFaceService#computeSha(String, long, Map)} and + * {@link ClarinHuggingFaceService#isValidRevision(String, String)}. + * + * @author DSpace at UFAL + */ +public class ClarinHuggingFaceServiceTest { + + private static final String UUID_1 = "11111111-1111-1111-1111-111111111111"; + + private static final String UUID_2 = "22222222-2222-2222-2222-222222222222"; + + private static final long LAST_MODIFIED_1 = 1_700_000_000_000L; + + private static final long LAST_MODIFIED_2 = 1_800_000_000_000L; + + /** + * Build a sample filename-to-md5 map. + * + * @return a map with two entries + */ + private Map sampleFiles() { + Map files = new LinkedHashMap<>(); + files.put("model.bin", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + files.put("config.json", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + return files; + } + + @Test + public void testComputeShaReturns40CharLowercaseHex() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertTrue("sha should match [0-9a-f]{40} but was: " + sha, sha.matches("[0-9a-f]{40}")); + } + + @Test + public void testComputeShaIsDeterministic() { + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertEquals(sha1, sha2); + } + + @Test + public void testComputeShaIsInsensitiveToMapInsertionOrder() { + Map hashMapOrder = new HashMap<>(); + hashMapOrder.put("model.bin", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + hashMapOrder.put("config.json", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + hashMapOrder.put("tokenizer.json", "cccccccccccccccccccccccccccccccc"); + + Map reversedLinkedOrder = new LinkedHashMap<>(); + reversedLinkedOrder.put("tokenizer.json", "cccccccccccccccccccccccccccccccc"); + reversedLinkedOrder.put("config.json", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + reversedLinkedOrder.put("model.bin", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + + String shaFromHashMap = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, hashMapOrder); + String shaFromReversedLinkedMap = + ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, reversedLinkedOrder); + + assertEquals(shaFromHashMap, shaFromReversedLinkedMap); + } + + @Test + public void testComputeShaChangesWhenUuidChanges() { + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_2, LAST_MODIFIED_1, sampleFiles()); + assertNotEquals(sha1, sha2); + } + + @Test + public void testComputeShaChangesWhenLastModifiedChanges() { + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_2, sampleFiles()); + assertNotEquals(sha1, sha2); + } + + @Test + public void testComputeShaChangesWhenFilenameChanges() { + Map files = sampleFiles(); + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, files); + + Map renamed = new LinkedHashMap<>(); + renamed.put("model-renamed.bin", files.get("model.bin")); + renamed.put("config.json", files.get("config.json")); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, renamed); + + assertNotEquals(sha1, sha2); + } + + @Test + public void testComputeShaChangesWhenMd5Changes() { + Map files = sampleFiles(); + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, files); + + Map changed = new LinkedHashMap<>(files); + changed.put("model.bin", "ffffffffffffffffffffffffffffffff"); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, changed); + + assertNotEquals(sha1, sha2); + } + + @Test + public void testComputeShaChangesWhenFileAdded() { + Map files = sampleFiles(); + String sha1 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, files); + + Map withExtra = new LinkedHashMap<>(files); + withExtra.put("extra.txt", "dddddddddddddddddddddddddddddddd"); + String sha2 = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, withExtra); + + assertNotEquals(sha1, sha2); + } + + @Test + public void testIsValidRevisionMainIsAlwaysValid() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertTrue(ClarinHuggingFaceService.isValidRevision("main", sha)); + } + + @Test + public void testIsValidRevisionMatchingShaIsValid() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertTrue(ClarinHuggingFaceService.isValidRevision(sha, sha)); + } + + @Test + public void testIsValidRevisionMatchingShaIsCaseInsensitive() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertTrue(ClarinHuggingFaceService.isValidRevision(sha.toUpperCase(), sha)); + } + + @Test + public void testIsValidRevisionDifferentShaIsInvalid() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + String otherSha = ClarinHuggingFaceService.computeSha(UUID_2, LAST_MODIFIED_2, sampleFiles()); + assertFalse(ClarinHuggingFaceService.isValidRevision(otherSha, sha)); + } + + @Test + public void testIsValidRevisionNullIsInvalid() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertFalse(ClarinHuggingFaceService.isValidRevision(null, sha)); + } + + @Test + public void testIsValidRevisionEmptyIsInvalid() { + String sha = ClarinHuggingFaceService.computeSha(UUID_1, LAST_MODIFIED_1, sampleFiles()); + assertFalse(ClarinHuggingFaceService.isValidRevision("", sha)); + } +} diff --git a/dspace/config/clarin-dspace.cfg b/dspace/config/clarin-dspace.cfg index a481beb856aa..312f2e06f3bb 100644 --- a/dspace/config/clarin-dspace.cfg +++ b/dspace/config/clarin-dspace.cfg @@ -344,3 +344,25 @@ user.registration = false # This option allows submitter to skip file upload step in the submission process webui.submit.upload.required = false + +#------------------------------------------------------------------# +#-------------------------HUGGINGFACE FACADE------------------------# +#------------------------------------------------------------------# +# HuggingFace-Hub-compatible read-only API facade, allowing tools built on the +# huggingface_hub Python client (e.g. `from_pretrained`) to resolve DSpace items as +# "models". An item is exposed through the facade only if it passes the exposure +# filter below (metadata field/value match) or belongs to one of the listed +# collections; if neither is configured, no items are exposed. + +# Master switch for the HuggingFace-Hub-compatible facade. +hf.api.enabled = false + +# Metadata field used to select items exposed through the HuggingFace facade. +hf.api.filter.metadata-field = dc.type + +# Value that `hf.api.filter.metadata-field` must equal for an item to be exposed. +hf.api.filter.value = machineLearningModel + +# Comma-separated list of collection handles whose items are exposed through the +# HuggingFace facade, regardless of the metadata filter above. +hf.api.exposed.collections = From 863a1972528b1cd1d1b574bf79ce1d485310b2a4 Mon Sep 17 00:00:00 2001 From: Ondrej Kosarko Date: Thu, 9 Jul 2026 09:53:57 +0200 Subject: [PATCH 2/2] Fix: enforce READ authorization in HuggingFace /api/models listing Copilot review on PR #1381 flagged that listModels() returned metadata for every exposed item without checking READ on the Item, letting anonymous users see the existence/title/tags of embargoed or private items via the facade's model listing. ClarinHuggingFaceService#findExposedItems() now skips any candidate the current context user cannot READ (authorizeActionBoolean), applied before the result is capped at the requested limit so unauthorized items never count against pagination. This mirrors the READ check already done by resolveAuthorizedRepository()/resolveFile() for the single-model and resolve/download endpoints, keeping all HF facade read paths consistent. Adds ClarinHuggingFaceControllerIT#listModelsExcludesItemsWithoutReadAccess covering the anonymous vs. admin visibility of a restricted-but-exposed item. --- .../rest/hf/ClarinHuggingFaceController.java | 10 ++++--- .../app/rest/hf/ClarinHuggingFaceService.java | 18 +++++++++-- .../rest/ClarinHuggingFaceControllerIT.java | 30 +++++++++++++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java index 169b1b4ac94e..329a4ecd75be 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceController.java @@ -206,10 +206,12 @@ public ResponseEntity tree(@PathVariable String prefix, @PathVariable St /** * List exposed Items ("models"), HuggingFace-Hub {@code GET /api/models} style. * - *

This is a metadata-only listing of publicly exposed items: unlike the model-info/tree/resolve - * endpoints, no per-item {@code READ} authorization check is performed, since only items that already - * pass {@link ClarinHuggingFaceService#isExposed(Context, Item)} (fail-closed, public-by-configuration) - * are ever returned.

+ *

This is a metadata-only listing of exposed items: only items that pass + * {@link ClarinHuggingFaceService#isExposed(Context, Item)} (fail-closed, public-by-configuration) AND + * that the current context user is authorized to {@code READ} are ever returned -- the {@code READ} + * check is performed by {@link ClarinHuggingFaceService#findExposedItems(Context, String, int)} itself, + * before pagination, so that embargoed/private items never count towards {@code limit} and are never + * leaked to a user who could not otherwise read them.

* * @param search an optional, case-insensitive substring to match against the item title * @param limit the maximum number of items to return (default 20, capped at 100) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java index 3617e1c739fc..bdf34f09a91e 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/hf/ClarinHuggingFaceService.java @@ -25,6 +25,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.dspace.authorize.AuthorizeException; +import org.dspace.authorize.service.AuthorizeService; import org.dspace.content.Bitstream; import org.dspace.content.Bundle; import org.dspace.content.Collection; @@ -93,6 +94,9 @@ public class ClarinHuggingFaceService { @Autowired private HandleService handleService; + @Autowired + private AuthorizeService authorizeService; + /** * Master switch for the HuggingFace-Hub-compatible facade. * @@ -201,15 +205,20 @@ public Item resolveExposedItem(Context context, String handle) throws SQLExcepti * * *

Every candidate is (re-)checked against {@link #isExposed(Context, Item)}, fail-closed, which also - * filters out withdrawn/non-archived items that may still be reachable through the collection path.

+ * filters out withdrawn/non-archived items that may still be reachable through the collection path. Each + * exposed candidate is additionally checked for {@code READ} authorization for the current context user + * ({@link AuthorizeService#authorizeActionBoolean(Context, DSpaceObject, int)}) before it counts towards + * {@code limit}, so that embargoed/private items are filtered out before pagination rather than after, + * and are never returned to a user who could not otherwise read them.

* * @param context the DSpace context * @param search an optional, case-insensitive substring to match against the item title (or name, if * the item has no title); {@code null} or blank to not filter by title * @param limit the maximum number of items to return; a non-positive value defaults to 20, and any * value is capped at 100 - * @return the list of exposed items matching {@code search}, capped at the effective limit; empty if - * neither exposure mechanism is configured, or no item is exposed and matches + * @return the list of exposed, READ-authorized items matching {@code search}, capped at the effective + * limit; empty if neither exposure mechanism is configured, or no item is exposed, readable and + * matches * @throws SQLException if a database error occurs */ public List findExposedItems(Context context, String search, int limit) throws SQLException { @@ -265,6 +274,9 @@ public List findExposedItems(Context context, String search, int limit) th if (searchLower != null && !matchesSearch(item, searchLower)) { continue; } + if (!authorizeService.authorizeActionBoolean(context, item, Constants.READ)) { + continue; + } result.add(item); } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java index 162d0ab52365..87427d514954 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinHuggingFaceControllerIT.java @@ -217,6 +217,36 @@ public void listModelsReturnsOnlyExposedItems() throws Exception { .andExpect(jsonPath("$[*].id", not(hasItem(nonExposedItem.getHandle())))); } + @Test + public void listModelsExcludesItemsWithoutReadAccess() throws Exception { + context.turnOffAuthorisationSystem(); + Collection collection = exposedItem.getOwningCollection(); + Item restrictedExposedItem = ItemBuilder.createItem(context, collection) + .withTitle("Restricted Machine Learning Model") + .withType("machineLearningModel") + .build(); + // Remove all read policies from the item and grant READ to admin only, so the item is still + // "exposed" (passes the HuggingFace filter) but not readable by an anonymous user. + authorizeService.removeAllPolicies(context, restrictedExposedItem); + ResourcePolicyBuilder.createResourcePolicy(context, admin, null) + .withDspaceObject(restrictedExposedItem) + .withAction(Constants.READ) + .build(); + context.restoreAuthSystemState(); + + // Anonymous user cannot read the restricted item, so it must not appear in the listing. + getClient().perform(get(ENDPOINT_BASE)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$[*].id", hasItem(exposedItem.getHandle()))) + .andExpect(jsonPath("$[*].id", not(hasItem(restrictedExposedItem.getHandle())))); + + // The admin can read it, so it must appear in the listing. + String adminToken = getAuthToken(admin.getEmail(), password); + getClient(adminToken).perform(get(ENDPOINT_BASE)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$[*].id", hasItem(restrictedExposedItem.getHandle()))); + } + @Test public void listModelsSupportsSearchAndLimit() throws Exception { context.turnOffAuthorisationSystem();