Add Docker setup

Dockerfile

@@ -0,0 +1,29 @@
+FROM maven:3.9-eclipse-temurin-21-alpine AS build
+
+RUN mkdir -p /app
+WORKDIR /app
+COPY . /app/burgerlinker
+
+RUN if [ ! -f /app/burgerlinker/pom.xml ]; then \
+      rm -rf burgerlinker; \
+      git clone https://github.com/CLARIAH/burgerLinker.git; \
+    fi
+
+RUN cd /app/burgerlinker && \
+    mvn install
+
+FROM eclipse-temurin:21-jre-alpine
+
+RUN apk update && apk upgrade && \
+    apk add --no-cache python3 py3-pandas
+
+RUN mkdir -p /app
+WORKDIR /app
+
+COPY --from=build /app/burgerlinker/target/burgerLinker-0.0.1-SNAPSHOT-jar-with-dependencies.jar /app/
+COPY --from=build /app/burgerlinker/assets/docker/entrypoint.sh /app/
+COPY --from=build /app/burgerlinker/assets/csv-to-rdf/zeeland-dataset/script.py /app/convert-to-RDF.py
+
+ENV JAVA_OPTS=""
+
+ENTRYPOINT ["/app/entrypoint.sh"]

README.md

@@ -211,3 +211,17 @@ It would be possible to add more general matching functionalities that are not d
 One possible way would be to provide a JSON Schema as an additional input to any given dataset, specifying the (i) Classes that the user wish to match their instances (e.g. sourceClass: iisg:Newborn ; targetClass: iisg:Groom), and the (ii) Properties that should be considered in the matching (e.g. schema:givenName; schema:familyName).
 
 Subsequently, the fast matching algorithm could be used for many other linkage purposes (in Digital Humanities), e.g. places, occupations and products.
+
+---
+
+## Docker setup
+
+A Docker image can be created from a checkout or directly from GitHub:
+
+```docker build -t burgerlinker https://raw.githubusercontent.com/CLARIAH/burgerLinker/main/Dockerfile```
+
+The docker image supports the complete pipeline as mentioned above, starting from the CSV files `registrations.csv` and `persons.csv` which it expects in a `CSV` subdirectory of your dataset, e.g. `/path-to/my-dataset/CSV`. To start linking:
+
+```docker -e JAVA_OPTS="-Xms128g -Xmx192g" run -v /path-to:/data burgerlinker my-dataset --function between_m_m --maxLev 1```
+
+It will put the result of the RDF and HDT conversions into the `RDF` subdirectory, e.g. `/path-to/my-dataset/RDF`. The result directories and a log file are created next to the `CSV` and `RDF` subdirectories. Running time and memory consumption depend on the input size, but creating the HDT index takes quite some time and memory (tune the `JAVA_OPTS=-Xms128g -Xmx192g` as needed).

assets/csv-to-rdf/zeeland-dataset/script.py

@@ -7,6 +7,7 @@
 import glob
 import collections
 import re
+import sys
 from datetime import datetime
 
 
@@ -296,7 +297,7 @@ def convertPersonsToRDF(inputData, outputData):
     start_time = datetime.now()
     f = open(outputData,"w+")
     ch_size = 10000
-    df_chunk = pd.read_csv(inputData, chunksize=ch_size, sep=";", low_memory=False, error_bad_lines=False, keep_default_na=False, encoding = 'latin-1')
+    df_chunk = pd.read_csv(inputData, chunksize=ch_size, sep=";", low_memory=False, on_bad_lines='skip', keep_default_na=False, encoding = 'latin-1')
     counter = 0
     for chunk in df_chunk:
         print("# " + str(counter) + " rows")
@@ -366,11 +367,22 @@ def convertPersonsToRDF(inputData, outputData):
     time_elapsed = datetime.now() - start_time
     print('Time elapsed (hh:mm:ss) {}'.format(time_elapsed))
 
-registrations_csv_path = "registrations.csv"
-output_file_registrations = "registrations.nq"
+
+if len(sys.argv) >= 3:
+    indir = sys.argv[1]
+    outdir = sys.argv[2]
+elif len(sys.argv) >= 2:
+    indir = sys.argv[1]
+    outdir = indir
+else:
+    indir =  "."
+    outdir = "."
+
+registrations_csv_path = indir + "/registrations.csv"
+output_file_registrations = outdir + "/registrations.nq"
 convertRegistrationsToRDF(registrations_csv_path, output_file_registrations)
-persons_csv_path = "persons.csv"
-output_file_persons = "persons.nq"
+persons_csv_path = indir + "/persons.csv"
+output_file_persons = outdir + "/persons.nq"
 convertPersonsToRDF(persons_csv_path, output_file_persons)
 
 # using the terminal, you can later merge the resulting files using the following cat command:

assets/docker/entrypoint.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+
+DATA="/data"
+
+if [ -z $1 ]; then
+    echo "!ERR: no dataset name provided!"
+    exit 1
+fi
+
+DS="$1"
+LOG="${DATA}/${DS}/${DS}.log"
+
+echo "BEGIN[`date`] ${*}" >> ${LOG}
+
+if [ ! -d ${DATA}/${DS} ]; then
+    echo "!ERR: dataset directory doesn't exist!" 2>&1 | tee -a ${LOG}
+    exit 2
+fi
+
+if [ ! -d ${DATA}/${DS}/RDF ]; then
+    mkdir -p ${DATA}/${DS}/RDF 2>&1 | tee -a ${LOG}
+fi
+
+if [ ! -f ${DATA}/${DS}/RDF/${DS}.hdt ]; then
+    echo "?INF: HDT doesn't exist yet" 2>&1 | tee -a ${LOG}
+
+    if [ ! -f ${DATA}/${DS}/RDF/${DS}.nq ]; then
+        echo "?INF: NQ doesn't exist yet" 2>&1 | tee -a ${LOG}
+
+        if [ ! -d ${DATA}/${DS}/CSV ]; then
+            echo "!ERR: dataset CSV directory doesn't exist!" 2>&1 | tee -a ${LOG}
+            exit 3
+        fi
+
+        # turn CSV into NQ
+        echo "?INF: generate NQ"  2>&1 | tee -a ${LOG}
+        python3 /app/convert-to-RDF.py ${DATA}/${DS}/CSV ${DATA}/${DS}/RDF 2>&1 | tee -a ${LOG}
+        ERR="$?"
+        if [ ${ERR} -ne 0 ]; then
+            exit ${ERR}
+        fi
+
+        cat ${DATA}/${DS}/RDF/*.nq > ${DATA}/${DS}/RDF/${DS}.nq
+    fi
+
+    # turn NQ into HDT
+    echo "?INF: generate HDT"  2>&1 | tee -a ${LOG}
+    java $JAVA_OPTS -jar burgerLinker-0.0.1-SNAPSHOT-jar-with-dependencies.jar --inputData ${DATA}/${DS}/RDF/${DS}.nq --outputDir ${DATA}/${DS}/RDF --function convertToHDT 2>&1 | tee -a ${LOG}
+    ERR="${?}"
+    if [ ${ERR} -ne 0 ]; then
+        exit ${ERR}
+    fi
+fi
+
+# get rid of $DS in $1
+shift
+
+java $JAVA_OPTS -jar burgerLinker-0.0.1-SNAPSHOT-jar-with-dependencies.jar --inputData ${DATA}/${DS}/RDF/${DS}.hdt --outputDir ${DATA}/${DS} $* 2>&1 | tee -a ${LOG}
+ERR="${?}"
+echo " END [`date`] ${*}" >> ${LOG}
+
+exit $ERR

pom.xml

@@ -13,8 +13,8 @@
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<maven.compiler.source>3.8</maven.compiler.source>
-		<maven.compiler.target>3.8</maven.compiler.target>
+		<maven.compiler.source>11</maven.compiler.source>
+		<maven.compiler.target>11</maven.compiler.target>
 	</properties>
     <build>
 		<plugins>
@@ -29,9 +29,12 @@
 						</goals>
 						<configuration>
 							<archive>
+								<manifestEntries>
+									<Multi-Release>true</Multi-Release>
+								</manifestEntries>
 								<manifest>
 									<mainClass>
-										iisg.amsterdam.burgerLinker.App
+										iisg.amsterdam.burgerlinker.App
 									</mainClass>
 								</manifest>
 							</archive>
@@ -75,7 +78,7 @@
 		<dependency>
 			<groupId>org.rdfhdt</groupId>
 			<artifactId>hdt-java-core</artifactId>
-			<version>3.0.9</version>
+			<version>3.0.10</version>
 		</dependency>
 <!-- 		<dependency>
 			<groupId>org.rdfhdt</groupId>
@@ -105,5 +108,11 @@
 			<artifactId>opencsv</artifactId>
 			<version>5.2</version>
 		</dependency>
+		<dependency>
+			<groupId>org.projectlombok</groupId>
+			<artifactId>lombok</artifactId>
+			<version>1.18.32</version>
+			<scope>provided</scope>
+		</dependency>
 	</dependencies>
 </project>