diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..ca482dc
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,29 @@
+FROM maven:3.9-eclipse-temurin-21-alpine AS build
+
+RUN mkdir -p /app
+WORKDIR /app
+COPY . /app/burgerlinker
+
+RUN if [ ! -f /app/burgerlinker/pom.xml ]; then \
+ rm -rf burgerlinker; \
+ git clone https://github.com/CLARIAH/burgerLinker.git; \
+ fi
+
+RUN cd /app/burgerlinker && \
+ mvn install
+
+FROM eclipse-temurin:21-jre-alpine
+
+RUN apk update && apk upgrade && \
+ apk add --no-cache python3 py3-pandas
+
+RUN mkdir -p /app
+WORKDIR /app
+
+COPY --from=build /app/burgerlinker/target/burgerLinker-0.0.1-SNAPSHOT-jar-with-dependencies.jar /app/
+COPY --from=build /app/burgerlinker/assets/docker/entrypoint.sh /app/
+COPY --from=build /app/burgerlinker/assets/csv-to-rdf/zeeland-dataset/script.py /app/convert-to-RDF.py
+
+ENV JAVA_OPTS=""
+
+ENTRYPOINT ["/app/entrypoint.sh"]
diff --git a/README.md b/README.md
index 674abdb..b5d1ba0 100755
--- a/README.md
+++ b/README.md
@@ -211,3 +211,17 @@ It would be possible to add more general matching functionalities that are not d
One possible way would be to provide a JSON Schema as an additional input to any given dataset, specifying the (i) Classes that the user wish to match their instances (e.g. sourceClass: iisg:Newborn ; targetClass: iisg:Groom), and the (ii) Properties that should be considered in the matching (e.g. schema:givenName; schema:familyName).
Subsequently, the fast matching algorithm could be used for many other linkage purposes (in Digital Humanities), e.g. places, occupations and products.
+
+---
+
+## Docker setup
+
+A Docker image can be created from a checkout or directly from GitHub:
+
+```docker build -t burgerlinker https://raw.githubusercontent.com/CLARIAH/burgerLinker/main/Dockerfile```
+
+The docker image supports the complete pipeline as mentioned above, starting from the CSV files `registrations.csv` and `persons.csv` which it expects in a `CSV` subdirectory of your dataset, e.g. `/path-to/my-dataset/CSV`. To start linking:
+
+```docker -e JAVA_OPTS="-Xms128g -Xmx192g" run -v /path-to:/data burgerlinker my-dataset --function between_m_m --maxLev 1```
+
+It will put the result of the RDF and HDT conversions into the `RDF` subdirectory, e.g. `/path-to/my-dataset/RDF`. The result directories and a log file are created next to the `CSV` and `RDF` subdirectories. Running time and memory consumption depend on the input size, but creating the HDT index takes quite some time and memory (tune the `JAVA_OPTS=-Xms128g -Xmx192g` as needed).
diff --git a/assets/csv-to-rdf/zeeland-dataset/script.py b/assets/csv-to-rdf/zeeland-dataset/script.py
index 68693cd..756e0f0 100755
--- a/assets/csv-to-rdf/zeeland-dataset/script.py
+++ b/assets/csv-to-rdf/zeeland-dataset/script.py
@@ -7,6 +7,7 @@
import glob
import collections
import re
+import sys
from datetime import datetime
@@ -296,7 +297,7 @@ def convertPersonsToRDF(inputData, outputData):
start_time = datetime.now()
f = open(outputData,"w+")
ch_size = 10000
- df_chunk = pd.read_csv(inputData, chunksize=ch_size, sep=";", low_memory=False, error_bad_lines=False, keep_default_na=False, encoding = 'latin-1')
+ df_chunk = pd.read_csv(inputData, chunksize=ch_size, sep=";", low_memory=False, on_bad_lines='skip', keep_default_na=False, encoding = 'latin-1')
counter = 0
for chunk in df_chunk:
print("# " + str(counter) + " rows")
@@ -366,11 +367,22 @@ def convertPersonsToRDF(inputData, outputData):
time_elapsed = datetime.now() - start_time
print('Time elapsed (hh:mm:ss) {}'.format(time_elapsed))
-registrations_csv_path = "registrations.csv"
-output_file_registrations = "registrations.nq"
+
+if len(sys.argv) >= 3:
+ indir = sys.argv[1]
+ outdir = sys.argv[2]
+elif len(sys.argv) >= 2:
+ indir = sys.argv[1]
+ outdir = indir
+else:
+ indir = "."
+ outdir = "."
+
+registrations_csv_path = indir + "/registrations.csv"
+output_file_registrations = outdir + "/registrations.nq"
convertRegistrationsToRDF(registrations_csv_path, output_file_registrations)
-persons_csv_path = "persons.csv"
-output_file_persons = "persons.nq"
+persons_csv_path = indir + "/persons.csv"
+output_file_persons = outdir + "/persons.nq"
convertPersonsToRDF(persons_csv_path, output_file_persons)
# using the terminal, you can later merge the resulting files using the following cat command:
diff --git a/assets/docker/entrypoint.sh b/assets/docker/entrypoint.sh
new file mode 100755
index 0000000..f3c1276
--- /dev/null
+++ b/assets/docker/entrypoint.sh
@@ -0,0 +1,62 @@
+#!/bin/sh
+
+DATA="/data"
+
+if [ -z $1 ]; then
+ echo "!ERR: no dataset name provided!"
+ exit 1
+fi
+
+DS="$1"
+LOG="${DATA}/${DS}/${DS}.log"
+
+echo "BEGIN[`date`] ${*}" >> ${LOG}
+
+if [ ! -d ${DATA}/${DS} ]; then
+ echo "!ERR: dataset directory doesn't exist!" 2>&1 | tee -a ${LOG}
+ exit 2
+fi
+
+if [ ! -d ${DATA}/${DS}/RDF ]; then
+ mkdir -p ${DATA}/${DS}/RDF 2>&1 | tee -a ${LOG}
+fi
+
+if [ ! -f ${DATA}/${DS}/RDF/${DS}.hdt ]; then
+ echo "?INF: HDT doesn't exist yet" 2>&1 | tee -a ${LOG}
+
+ if [ ! -f ${DATA}/${DS}/RDF/${DS}.nq ]; then
+ echo "?INF: NQ doesn't exist yet" 2>&1 | tee -a ${LOG}
+
+ if [ ! -d ${DATA}/${DS}/CSV ]; then
+ echo "!ERR: dataset CSV directory doesn't exist!" 2>&1 | tee -a ${LOG}
+ exit 3
+ fi
+
+ # turn CSV into NQ
+ echo "?INF: generate NQ" 2>&1 | tee -a ${LOG}
+ python3 /app/convert-to-RDF.py ${DATA}/${DS}/CSV ${DATA}/${DS}/RDF 2>&1 | tee -a ${LOG}
+ ERR="$?"
+ if [ ${ERR} -ne 0 ]; then
+ exit ${ERR}
+ fi
+
+ cat ${DATA}/${DS}/RDF/*.nq > ${DATA}/${DS}/RDF/${DS}.nq
+ fi
+
+ # turn NQ into HDT
+ echo "?INF: generate HDT" 2>&1 | tee -a ${LOG}
+ java $JAVA_OPTS -jar burgerLinker-0.0.1-SNAPSHOT-jar-with-dependencies.jar --inputData ${DATA}/${DS}/RDF/${DS}.nq --outputDir ${DATA}/${DS}/RDF --function convertToHDT 2>&1 | tee -a ${LOG}
+ ERR="${?}"
+ if [ ${ERR} -ne 0 ]; then
+ exit ${ERR}
+ fi
+fi
+
+# get rid of $DS in $1
+shift
+
+java $JAVA_OPTS -jar burgerLinker-0.0.1-SNAPSHOT-jar-with-dependencies.jar --inputData ${DATA}/${DS}/RDF/${DS}.hdt --outputDir ${DATA}/${DS} $* 2>&1 | tee -a ${LOG}
+ERR="${?}"
+echo " END [`date`] ${*}" >> ${LOG}
+
+exit $ERR
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index d610d10..591b361 100755
--- a/pom.xml
+++ b/pom.xml
@@ -13,8 +13,8 @@
UTF-8
- 3.8
- 3.8
+ 11
+ 11
@@ -29,9 +29,12 @@
+
+ true
+
- iisg.amsterdam.burgerLinker.App
+ iisg.amsterdam.burgerlinker.App
@@ -75,7 +78,7 @@
org.rdfhdt
hdt-java-core
- 3.0.9
+ 3.0.10