Feature/reuse openccg

Dockerfile

@@ -5,18 +5,28 @@ LABEL description="A small webapp to parse sentences using the DiaSpace grammar
 LABEL version="2.1"
 
 ARG GRAMMAR_VERSION=master
-ARG OPENCCG_VERSION=0.9.5
+ARG OPENCCG_LIB_VERSION=0.9.5
+ARG OPENCCG_REPOSITORY=https://github.com/OpenCCG/openccg
+ARG OPENCCG_VERSION=master
+ARG WCCG_POOL_SIZE=3
 
 EXPOSE 5000 8080
 
 ENV OPENCCG_HOME /openccg
-ENV PATH "$OPENCCG_HOME/bin:$PATH"
-ENV LD_LIBRARY_PATH "$OPENCCG_HOME/lib:$LD_LIBRARY_PATH"
-
-# Download and extract OpenCCG
-RUN curl -o openccg-${OPENCCG_VERSION}.tgz https://datapacket.dl.sourceforge.net/project/openccg/openccg/openccg%20v${OPENCCG_VERSION}%20-%20deplen%2C%20kenlm%2C%20disjunctivizer/openccg-${OPENCCG_VERSION}.tgz \
-    && tar zxf openccg-${OPENCCG_VERSION}.tgz \
-    && rm openccg-${OPENCCG_VERSION}.tgz \
+ENV PATH "${OPENCCG_HOME}/bin:$PATH"
+ENV LD_LIBRARY_PATH "${OPENCCG_HOME}/lib:${LD_LIBRARY_PATH}"
+ENV WCCG_POOL_SIZE=${WCCG_POOL_SIZE}
+
+# Download and extract OpenCCG -- first for libraries, then the requested source-code version
+RUN curl -o openccg-${OPENCCG_LIB_VERSION}.tgz https://datapacket.dl.sourceforge.net/project/openccg/openccg/openccg%20v${OPENCCG_LIB_VERSION}%20-%20deplen%2C%20kenlm%2C%20disjunctivizer/openccg-${OPENCCG_LIB_VERSION}.tgz \
+    && tar zxf openccg-${OPENCCG_LIB_VERSION}.tgz \
+    && rm openccg-${OPENCCG_LIB_VERSION}.tgz \
+# Source code overwrites
+    && curl -o openccg.zip -L ${OPENCCG_REPOSITORY}/archive/${OPENCCG_VERSION}.zip \
+    && unzip openccg.zip \
+    && rm openccg.zip \
+    && cp -r openccg-*/* /openccg/ \
+    && rm -r openccg-* \
 # Download and extract grammar
     && curl -o grammar.zip -L https://github.com/shoeffner/openccg-gum-cooking/archive/${GRAMMAR_VERSION}.zip \
     && unzip -d /tmp grammar.zip \
@@ -34,6 +44,8 @@ RUN curl -o openccg-${OPENCCG_VERSION}.tgz https://datapacket.dl.sourceforge.net
                     uwsgi \
                     tatsu \
                     pygraphviz \
+                    pexpect \
+# Build OpenCCG
     && (cd /openccg && ccg-build)
 
 COPY setup.py requirements.txt README.md /app/

Jenkinsfile

@@ -4,7 +4,7 @@ pipeline {
     stages {
         stage('build') {
             steps {
-                sh 'docker build . --no-cache --build-arg GRAMMAR_VERSION=legacy/grammar -t web-openccg:$(git rev-parse --short HEAD)'
+                sh 'docker build . --no-cache --build-arg OPENCCG_REPOSITORY=https://github.com/shoeffner/openccg --build-arg OPENCCG_VERSION=feature/wccg-prompt --build-arg GRAMMAR_VERSION=legacy/grammar -t web-openccg:$(git rev-parse --short HEAD)'
             }
         }
 

Makefile

@@ -5,7 +5,11 @@ webopenccg/generated_openccg_parser.py: OpenCCG.ebnf
 
 .PHONY: build
 build:
-	docker build . --build-arg GRAMMAR_VERSION=legacy/grammar -t web-openccg
+	docker build . \
+		--build-arg OPENCCG_REPOSITORY=https://github.com/shoeffner/openccg \
+		--build-arg OPENCCG_VERSION=feature/wccg-prompt \
+		--build-arg GRAMMAR_VERSION=legacy/grammar \
+		-t web-openccg
 
 webopenccg/static/%.js:
 	docker run --rm --detach --name web-openccg-build-$(notdir $@) web-openccg

requirements.txt

@@ -2,3 +2,4 @@ requests
 tatsu
 pygraphviz
 flask
+pexpect

webopenccg/wccg.py

@@ -1,15 +1,58 @@
 import itertools
+import multiprocessing
 import os
 import re
 import string
-import subprocess
+import time
+
+from pexpect.replwrap import REPLWrapper, PEXPECT_PROMPT
 
 from tatsu.util import asjson
 from tatsu.model import ModelBuilderSemantics
 
 from webopenccg.generated_openccg_parser import OpenCCGParser
 
 
+class WCCGProcess:
+    """Holds a wccg process to be reused."""
+
+    def __init__(self):
+        self.mutex = multiprocessing.Lock()
+        self._new_repl()
+
+    def _new_repl(self):
+        self.repl = REPLWrapper(f'wccg -prompt {PEXPECT_PROMPT} -showsem -showall {os.environ.get("GRAMMAR_DIR", "/grammar")}', PEXPECT_PROMPT, None)
+
+    def parse(self, sentence):
+        """Parse a sentence by passing it to the wccg process and read the results.
+        This is thread-safe, so multiple processes can try to access the process.
+        """
+        with self.mutex:
+            return self.repl.run_command(sentence)
+
+
+class WCCGPool:
+    def __init__(self, pool_size=3):
+        self.mutex = multiprocessing.Lock()
+        self.processes = [WCCGProcess()]
+        self.current_idx = 0
+        self.pool_size = pool_size
+
+        self.last_activity = 0
+        self.activity_threshold = 3
+
+    def __next__(self):
+        # TODO(@shoeffner): The pool currently never shrinks back
+        with self.mutex:
+            now = time.time()
+            if now - self.last_activity < self.activity_threshold and len(self.processes) < self.pool_size:
+                self.processes.append(WCCGProcess())
+            self.last_activity = now
+
+            self.current_idx = (self.current_idx + 1) % len(self.processes)
+            return self.processes[self.current_idx]
+
+
 def parse(sentence):
     """Parses a sentence using OpenCCG's command line tool wccg.
 
@@ -26,13 +69,11 @@ def parse(sentence):
     if not sentence:
         return dict(error='No sentence provided.', http_status=400)
 
-    wccg_proc = subprocess.Popen(['wccg', '-showsem', '-showall', os.environ.get('GRAMMAR_DIR', '/grammar')],
-                                 stdin=subprocess.PIPE,
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.DEVNULL,
-                                 universal_newlines=True)
     sentence = re.sub(f'[{re.escape(string.punctuation)}]', '', sentence).lower()
-    response = wccg_proc.communicate(input=sentence)[0]
+
+    if not hasattr(parse, 'pool'):
+        parse.pool = WCCGPool(int(os.environ.get('WCCG_POOL_SIZE', 3)))
+    response = next(parse.pool).parse(sentence)
 
     wccg_response = _as_dict(response or f'"{sentence}": Unable to parse. wccg returned an empty response.')
     wccg_response = jsonify_parses(wccg_response)