diff --git a/Dockerfile b/Dockerfile index e41d7fe..f8d9220 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,18 +5,28 @@ LABEL description="A small webapp to parse sentences using the DiaSpace grammar LABEL version="2.1" ARG GRAMMAR_VERSION=master -ARG OPENCCG_VERSION=0.9.5 +ARG OPENCCG_LIB_VERSION=0.9.5 +ARG OPENCCG_REPOSITORY=https://github.com/OpenCCG/openccg +ARG OPENCCG_VERSION=master +ARG WCCG_POOL_SIZE=3 EXPOSE 5000 8080 ENV OPENCCG_HOME /openccg -ENV PATH "$OPENCCG_HOME/bin:$PATH" -ENV LD_LIBRARY_PATH "$OPENCCG_HOME/lib:$LD_LIBRARY_PATH" - -# Download and extract OpenCCG -RUN curl -o openccg-${OPENCCG_VERSION}.tgz https://datapacket.dl.sourceforge.net/project/openccg/openccg/openccg%20v${OPENCCG_VERSION}%20-%20deplen%2C%20kenlm%2C%20disjunctivizer/openccg-${OPENCCG_VERSION}.tgz \ - && tar zxf openccg-${OPENCCG_VERSION}.tgz \ - && rm openccg-${OPENCCG_VERSION}.tgz \ +ENV PATH "${OPENCCG_HOME}/bin:$PATH" +ENV LD_LIBRARY_PATH "${OPENCCG_HOME}/lib:${LD_LIBRARY_PATH}" +ENV WCCG_POOL_SIZE=${WCCG_POOL_SIZE} + +# Download and extract OpenCCG -- first for libraries, then the requested source-code version +RUN curl -o openccg-${OPENCCG_LIB_VERSION}.tgz https://datapacket.dl.sourceforge.net/project/openccg/openccg/openccg%20v${OPENCCG_LIB_VERSION}%20-%20deplen%2C%20kenlm%2C%20disjunctivizer/openccg-${OPENCCG_LIB_VERSION}.tgz \ + && tar zxf openccg-${OPENCCG_LIB_VERSION}.tgz \ + && rm openccg-${OPENCCG_LIB_VERSION}.tgz \ +# Source code overwrites + && curl -o openccg.zip -L ${OPENCCG_REPOSITORY}/archive/${OPENCCG_VERSION}.zip \ + && unzip openccg.zip \ + && rm openccg.zip \ + && cp -r openccg-*/* /openccg/ \ + && rm -r openccg-* \ # Download and extract grammar && curl -o grammar.zip -L https://github.com/shoeffner/openccg-gum-cooking/archive/${GRAMMAR_VERSION}.zip \ && unzip -d /tmp grammar.zip \ @@ -34,6 +44,8 @@ RUN curl -o openccg-${OPENCCG_VERSION}.tgz https://datapacket.dl.sourceforge.net uwsgi \ tatsu \ pygraphviz \ + pexpect \ +# Build OpenCCG && (cd /openccg && ccg-build) COPY setup.py requirements.txt README.md /app/ diff --git a/Jenkinsfile b/Jenkinsfile index b26571b..f828146 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,7 @@ pipeline { stages { stage('build') { steps { - sh 'docker build . --no-cache --build-arg GRAMMAR_VERSION=legacy/grammar -t web-openccg:$(git rev-parse --short HEAD)' + sh 'docker build . --no-cache --build-arg OPENCCG_REPOSITORY=https://github.com/shoeffner/openccg --build-arg OPENCCG_VERSION=feature/wccg-prompt --build-arg GRAMMAR_VERSION=legacy/grammar -t web-openccg:$(git rev-parse --short HEAD)' } } diff --git a/Makefile b/Makefile index cce6511..137251a 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,11 @@ webopenccg/generated_openccg_parser.py: OpenCCG.ebnf .PHONY: build build: - docker build . --build-arg GRAMMAR_VERSION=legacy/grammar -t web-openccg + docker build . \ + --build-arg OPENCCG_REPOSITORY=https://github.com/shoeffner/openccg \ + --build-arg OPENCCG_VERSION=feature/wccg-prompt \ + --build-arg GRAMMAR_VERSION=legacy/grammar \ + -t web-openccg webopenccg/static/%.js: docker run --rm --detach --name web-openccg-build-$(notdir $@) web-openccg diff --git a/requirements.txt b/requirements.txt index b4f6607..5190cdc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ requests tatsu pygraphviz flask +pexpect diff --git a/webopenccg/wccg.py b/webopenccg/wccg.py index 074fcfa..9875217 100644 --- a/webopenccg/wccg.py +++ b/webopenccg/wccg.py @@ -1,8 +1,11 @@ import itertools +import multiprocessing import os import re import string -import subprocess +import time + +from pexpect.replwrap import REPLWrapper, PEXPECT_PROMPT from tatsu.util import asjson from tatsu.model import ModelBuilderSemantics @@ -10,6 +13,46 @@ from webopenccg.generated_openccg_parser import OpenCCGParser +class WCCGProcess: + """Holds a wccg process to be reused.""" + + def __init__(self): + self.mutex = multiprocessing.Lock() + self._new_repl() + + def _new_repl(self): + self.repl = REPLWrapper(f'wccg -prompt {PEXPECT_PROMPT} -showsem -showall {os.environ.get("GRAMMAR_DIR", "/grammar")}', PEXPECT_PROMPT, None) + + def parse(self, sentence): + """Parse a sentence by passing it to the wccg process and read the results. + This is thread-safe, so multiple processes can try to access the process. + """ + with self.mutex: + return self.repl.run_command(sentence) + + +class WCCGPool: + def __init__(self, pool_size=3): + self.mutex = multiprocessing.Lock() + self.processes = [WCCGProcess()] + self.current_idx = 0 + self.pool_size = pool_size + + self.last_activity = 0 + self.activity_threshold = 3 + + def __next__(self): + # TODO(@shoeffner): The pool currently never shrinks back + with self.mutex: + now = time.time() + if now - self.last_activity < self.activity_threshold and len(self.processes) < self.pool_size: + self.processes.append(WCCGProcess()) + self.last_activity = now + + self.current_idx = (self.current_idx + 1) % len(self.processes) + return self.processes[self.current_idx] + + def parse(sentence): """Parses a sentence using OpenCCG's command line tool wccg. @@ -26,13 +69,11 @@ def parse(sentence): if not sentence: return dict(error='No sentence provided.', http_status=400) - wccg_proc = subprocess.Popen(['wccg', '-showsem', '-showall', os.environ.get('GRAMMAR_DIR', '/grammar')], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - universal_newlines=True) sentence = re.sub(f'[{re.escape(string.punctuation)}]', '', sentence).lower() - response = wccg_proc.communicate(input=sentence)[0] + + if not hasattr(parse, 'pool'): + parse.pool = WCCGPool(int(os.environ.get('WCCG_POOL_SIZE', 3))) + response = next(parse.pool).parse(sentence) wccg_response = _as_dict(response or f'"{sentence}": Unable to parse. wccg returned an empty response.') wccg_response = jsonify_parses(wccg_response)