From 7cc3e03ced910e0ff4fbf66afbe63cccb23ac429 Mon Sep 17 00:00:00 2001 From: kerim1 Date: Tue, 16 Apr 2024 16:08:13 +0200 Subject: [PATCH 1/2] Add Docker setup --- Dockerfile | 27 +++++++++ README.md | 14 +++++ assets/csv-to-rdf/zeeland-dataset/script.py | 20 +++++-- assets/docker/entrypoint.sh | 62 +++++++++++++++++++++ pom.xml | 17 ++++-- 5 files changed, 132 insertions(+), 8 deletions(-) create mode 100644 Dockerfile create mode 100755 assets/docker/entrypoint.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..652d42c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM maven:3.9-eclipse-temurin-21-alpine AS build + +RUN mkdir -p /app +WORKDIR /app +COPY . /app/burgerlinker + +RUN if [ ! -f /app/burgerlinker/pom.xml ]; then \ + rm -rf burgerlinker; \ + git clone https://github.com/CLARIAH/burgerLinker.git; \ + fi + +RUN cd /app/burgerlinker && \ + mvn install + +FROM eclipse-temurin:21-jre-alpine + +RUN apk update && apk upgrade && \ + apk add --no-cache python3 py3-pandas + +RUN mkdir -p /app +WORKDIR /app + +COPY --from=build /app/burgerlinker/target/burgerLinker-0.0.1-SNAPSHOT-jar-with-dependencies.jar /app/ +COPY --from=build /app/burgerlinker/assets/docker/entrypoint.sh /app/ +COPY --from=build /app/burgerlinker/assets/csv-to-rdf/zeeland-dataset/script.py /app/convert-to-RDF.py + +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/README.md b/README.md index 674abdb..715098f 100755 --- a/README.md +++ b/README.md @@ -211,3 +211,17 @@ It would be possible to add more general matching functionalities that are not d One possible way would be to provide a JSON Schema as an additional input to any given dataset, specifying the (i) Classes that the user wish to match their instances (e.g. sourceClass: iisg:Newborn ; targetClass: iisg:Groom), and the (ii) Properties that should be considered in the matching (e.g. schema:givenName; schema:familyName). Subsequently, the fast matching algorithm could be used for many other linkage purposes (in Digital Humanities), e.g. places, occupations and products. + +--- + +## Docker setup + +A Docker image can be created from a checkout or directly from GitHub: + +```docker build -t burgerlinker https://raw.githubusercontent.com/CLARIAH/burgerLinker/main/Dockerfile``` + +The docker image supports the complete pipeline as mentioned above, starting from the CSV files `registrations.csv` and `persons.csv` which it expects in a `CSV` subdirectory of your dataset, e.g. `/path-to/my-dataset/CSV`. To start linking: + +```docker -m 16g run -v /path-to:/data burgerlinker my-dataset --function between_m_m --maxLev 1``` + +It will put the result of the RDF and HDT conversions into the `RDF` subdirectory, e.g. `/path-to/my-dataset/RDF`. The result directories and a log file are created next to the `CSV` and `RDF` subdirectories. Running time and memory consumption depend on the input size, but creating the HDT index takes quite some time and memory (tune the `-m 16g` as needed). diff --git a/assets/csv-to-rdf/zeeland-dataset/script.py b/assets/csv-to-rdf/zeeland-dataset/script.py index 68693cd..4f3b5ad 100755 --- a/assets/csv-to-rdf/zeeland-dataset/script.py +++ b/assets/csv-to-rdf/zeeland-dataset/script.py @@ -7,6 +7,7 @@ import glob import collections import re +import sys from datetime import datetime @@ -366,11 +367,22 @@ def convertPersonsToRDF(inputData, outputData): time_elapsed = datetime.now() - start_time print('Time elapsed (hh:mm:ss) {}'.format(time_elapsed)) -registrations_csv_path = "registrations.csv" -output_file_registrations = "registrations.nq" + +if len(sys.argv) >= 3: + indir = sys.argv[1] + outdir = sys.argv[2] +elif len(sys.argv) >= 2: + indir = sys.argv[1] + outdir = indir +else: + indir = "." + outdir = "." + +registrations_csv_path = indir + "/registrations.csv" +output_file_registrations = outdir + "/registrations.nq" convertRegistrationsToRDF(registrations_csv_path, output_file_registrations) -persons_csv_path = "persons.csv" -output_file_persons = "persons.nq" +persons_csv_path = indir + "/persons.csv" +output_file_persons = outdir + "/persons.nq" convertPersonsToRDF(persons_csv_path, output_file_persons) # using the terminal, you can later merge the resulting files using the following cat command: diff --git a/assets/docker/entrypoint.sh b/assets/docker/entrypoint.sh new file mode 100755 index 0000000..cd0ff22 --- /dev/null +++ b/assets/docker/entrypoint.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +DATA="/data" + +if [ -z $1 ]; then + echo "!ERR: no dataset name provided!" + exit 1 +fi + +DS="$1" +LOG="${DATA}/${DS}/${DS}.log" + +echo "BEGIN[`date`] ${*}" >> ${LOG} + +if [ ! -d ${DATA}/${DS} ]; then + echo "!ERR: dataset directory doesn't exist!" 2>&1 | tee -a ${LOG} + exit 2 +fi + +if [ ! -d ${DATA}/${DS}/RDF ]; then + mkdir -p ${DATA}/${DS}/RDF 2>&1 | tee -a ${LOG} +fi + +if [ ! -f ${DATA}/${DS}/RDF/${DS}.hdt ]; then + echo "?INF: HDT doesn't exist yet" 2>&1 | tee -a ${LOG} + + if [ ! -f ${DATA}/${DS}/RDF/${DS}.nq ]; then + echo "?INF: NQ doesn't exist yet" 2>&1 | tee -a ${LOG} + + if [ ! -d ${DATA}/${DS}/CSV ]; then + echo "!ERR: dataset CSV directory doesn't exist!" 2>&1 | tee -a ${LOG} + exit 3 + fi + + # turn CSV into NQ + echo "?INF: generate NQ" 2>&1 | tee -a ${LOG} + python3 /app/convert-to-RDF.py ${DATA}/${DS}/CSV ${DATA}/${DS}/RDF 2>&1 | tee -a ${LOG} + ERR="$?" + if [ ${ERR} -ne 0 ]; then + exit ${ERR} + fi + + cat ${DATA}/${DS}/RDF/*.nq > ${DATA}/${DS}/RDF/${DS}.nq + fi + + # turn NQ into HDT + echo "?INF: generate HDT" 2>&1 | tee -a ${LOG} + java -jar burgerLinker-0.0.1-SNAPSHOT-jar-with-dependencies.jar --inputData ${DATA}/${DS}/RDF/${DS}.nq --outputDir ${DATA}/${DS}/RDF --function convertToHDT 2>&1 | tee -a ${LOG} + ERR="${?}" + if [ ${ERR} -ne 0 ]; then + exit ${ERR} + fi +fi + +# get rid of $DS in $1 +shift + +java -jar burgerLinker-0.0.1-SNAPSHOT-jar-with-dependencies.jar --inputData ${DATA}/${DS}/RDF/${DS}.hdt --outputDir ${DATA}/${DS} $* 2>&1 | tee -a ${LOG} +ERR="${?}" +echo " END [`date`] ${*}" >> ${LOG} + +exit $ERR \ No newline at end of file diff --git a/pom.xml b/pom.xml index d610d10..591b361 100755 --- a/pom.xml +++ b/pom.xml @@ -13,8 +13,8 @@ UTF-8 - 3.8 - 3.8 + 11 + 11 @@ -29,9 +29,12 @@ + + true + - iisg.amsterdam.burgerLinker.App + iisg.amsterdam.burgerlinker.App @@ -75,7 +78,7 @@ org.rdfhdt hdt-java-core - 3.0.9 + 3.0.10