Fix python runtime deadlock due to multithreading + fork (#97)

* Fix python runtime deadlock due to multithreading + fork

Drop support for Python2.7 runtime because it has reached End Of Life
and also because "spawn" and "forkserver" multiprocessing contexts are
not supported in Python 2.7 which are required to fix the deadlock.

* Update images

* Display K8s objects on test failure.

Co-authored-by: Andres Martinez Gotor <[email protected]>
Co-authored-by: Varun Basavaraj Patil <[email protected]>

Author: 3 people

.circleci/config.yml

@@ -10,6 +10,7 @@ defaults: &defaults
 run_test: &run_test
   steps:
     - checkout
+    - run: echo "export GIT_SHA1=${CIRCLE_SHA1}" >> $BASH_ENV
     - run: |
         # Determine if running the tests are necessary
         if [[ -n "$CIRCLE_PULL_REQUEST" ]]; then

script/integration-tests.sh

@@ -36,3 +36,20 @@ kubectl rollout status -n kubeless deployment/kubeless-controller-manager
 
 make -C ${ROOT_DIR}/${target} deploy
 make -C ${ROOT_DIR}/${target} test
+
+exit_code=$?
+
+# Just showing remaining k8s objects
+kubectl get all --all-namespaces
+
+if [ ${exit_code} -ne 0 -o -n "${TRAVIS_DUMP_LOGS}" ]; then
+    echo "INFO: Build ERRORed, dumping logs: ##"
+    for ns in kubeless default; do
+        echo "### LOGs: namespace: ${ns} ###"
+        kubectl get pod -n ${ns} -oname|xargs -I@ sh -xc "kubectl logs -n ${ns} @|sed 's|^|@: |'"
+    done
+    echo "INFO: Description"
+    kubectl describe pod -l created-by=kubeless
+fi
+[ ${exit_code} -eq 0 ] && echo "INFO: $0: SUCCESS" || echo "ERROR: $0: FAILED"
+exit ${exit_code}

stable/python/Dockerfile.2.7

@@ -17,9 +17,9 @@ RUN python ./get-pip.py
 RUN pip install bottle==0.12.13 cherrypy==8.9.1 wsgi-request-logger prometheus_client
 
 WORKDIR /
-ADD kubeless.py .
+ADD _kubeless.py .
 
 USER 1000
 
 ENV PYTHONUNBUFFERED 1
-CMD ["python", "/kubeless.py"]
+CMD ["python", "/_kubeless.py"]

stable/python/Dockerfile.3.6

@@ -17,9 +17,9 @@ RUN python ./get-pip.py
 RUN pip install bottle==0.12.13 cherrypy==8.9.1 wsgi-request-logger prometheus_client
 
 WORKDIR /
-ADD kubeless.py .
+ADD _kubeless.py .
 
 USER 1000
 
 ENV PYTHONUNBUFFERED 1
-CMD ["python", "/kubeless.py"]
+CMD ["python", "/_kubeless.py"]

stable/python/Dockerfile.3.7

@@ -1,77 +1,73 @@
+GIT_SHA1 ?= master
+BASE_URL := https://raw.githubusercontent.com/kubeless/runtimes/$(GIT_SHA1)
 
-build2.7:
-	docker build --pull --no-cache -t kubeless/python:2.7$$RUNTIME_TAG_MODIFIER -f Dockerfile.2.7 .
+# Wait until the Pod is in ready state. Without a readinessProbe it is not possible to
+# tell exactly when the Pod is ready to accept requests. So, adding additional sleep
+# (default = 2 seconds).
+define rollout-status
+  kubectl rollout status deployment/$(patsubst %-verify,%,$1) && sleep $(if $2,$2,2)
+endef
 
 build3.6:
 	docker build --pull --no-cache -t kubeless/python:3.6$$RUNTIME_TAG_MODIFIER -f Dockerfile.3.6 .
 
 build3.7:
 	docker build --pull --no-cache -t kubeless/python:3.7$$RUNTIME_TAG_MODIFIER -f Dockerfile.3.7 .
 
-push2.7:
-	docker push kubeless/python:2.7$$RUNTIME_TAG_MODIFIER
-
 push3.6:
 	docker push kubeless/python:3.6$$RUNTIME_TAG_MODIFIER
 
 push3.7:
 	docker push kubeless/python:3.7$$RUNTIME_TAG_MODIFIER
 
 # Mandatory jobs
-build-all: build2.7 build3.6 build3.7
-push-all: push2.7 push3.6 push3.7
+build-all: build3.6 build3.7
+push-all: push3.6 push3.7
 
 # Testing jobs
-deploy: get-python get-python-deps get-python-custom-port get-python-36 get-python-37 get-python-url-deps scheduled-get-python timeout-python get-python-secrets post-python post-python-custom-port custom-get-python
-test: get-python-verify get-python-deps-verify get-python-custom-port-verify get-python-36-verify get-python-37-verify get-python-url-deps-verify scheduled-get-python-verify timeout-python-verify get-python-secrets-verify custom-get-python-verify post-python-verify post-python-custom-port-verify
-
-get-python:
-	kubeless function deploy get-python --runtime python2.7 --handler helloget.foo --from-file examples/helloget.py
-
-get-python-verify:
-	kubectl rollout status deployment/get-python && sleep 2
-	kubeless function call get-python |egrep hello.world
-	kubeless function top --function get-python --out yaml |egrep total_calls.*[1-100000]
+deploy: get-python-deps get-python-custom-port get-python-36 get-python-37 get-python-url-deps scheduled-get-python timeout-python get-python-secrets post-python post-python-custom-port custom-get-python python-preload
+test: get-python-deps-verify get-python-custom-port-verify get-python-36-verify get-python-37-verify get-python-url-deps-verify scheduled-get-python-verify timeout-python-verify get-python-secrets-verify custom-get-python-verify post-python-verify post-python-custom-port-verify python-preload-verify
 
 get-python-deps:
-	cd examples && zip hellowithdeps.zip hellowithdeps.py  hellowithdepshelper.py && cd ..
-	kubeless function deploy get-python-deps --runtime python2.7 --handler hellowithdeps.foo --from-file examples/hellowithdeps.zip --dependencies examples/requirements.txt
+	cd examples && zip hellowithdeps.zip hellowithdeps.py hellowithdepshelper.py
+	kubeless function deploy get-python-deps --runtime python3.6 --handler hellowithdeps.foo --from-file examples/hellowithdeps.zip --dependencies examples/requirements.txt
 
 get-python-deps-verify:
-	kubectl rollout status deployment/get-python-deps && sleep 2
+	$(call rollout-status,$@,5)
 	kubeless function call get-python-deps |egrep Google
 
 get-python-custom-port:
-	kubeless function deploy get-python-custom-port --runtime python2.7 --handler helloget.foo --from-file examples/helloget.py --port 8081
+	kubeless function deploy get-python-custom-port --runtime python3.6 --handler helloget.foo --from-file examples/helloget.py --port 8081
 
 get-python-custom-port-verify:
-	kubectl rollout status deployment/get-python-custom-port && sleep 2
+	$(call rollout-status,$@)
 	kubectl get svc get-python-custom-port -o yaml | grep 'targetPort: 8081'
 	kubeless function call get-python-custom-port |egrep hello.world
 
 get-python-36:
 	kubeless function deploy get-python-36 --runtime python3.6 --handler helloget.foo --from-file examples/helloget.py
 
 get-python-36-verify:
-	kubectl rollout status deployment/get-python-36 && sleep 2
+	$(call rollout-status,$@)
 	kubeless function call get-python-36 |egrep hello.world
 
 get-python-37:
 	kubeless function deploy get-python-37 --runtime python3.7 --handler helloget.foo --from-file examples/helloget.py
 
 get-python-37-verify:
-	kubectl rollout status deployment/get-python-37 && sleep 2
+	$(call rollout-status,$@)
 	kubeless function call get-python-37 |egrep hello.world
 
 get-python-url-deps:
-	kubeless function deploy get-python-url-deps --runtime python2.7 --handler helloget.foo --from-file https://raw.githubusercontent.com/kubeless/kubeless/v1.0.0-alpha.1/examples/python/hellowithdeps.py --dependencies https://raw.githubusercontent.com/kubeless/kubeless/v1.0.0-alpha.1/examples/python/requirements.txt
+	cd examples && zip hellowithdeps.zip hellowithdeps.py hellowithdepshelper.py
+	kubeless function deploy get-python-url-deps --runtime python3.6 --handler hellowithdeps.foo --from-file examples/hellowithdeps.zip --dependencies $(BASE_URL)/stable/python/examples/requirements.txt
 
 get-python-url-deps-verify:
-	kubectl rollout status deployment/get-python-url-deps && sleep 2
+	$(call rollout-status,$@)
 	kubeless function call get-python-url-deps |egrep Google
 
 scheduled-get-python:
-	kubeless function deploy scheduled-get-python --schedule "* * * * *" --runtime python2.7 --handler helloget.foo --from-file examples/helloget.py
+	kubeless function deploy scheduled-get-python --schedule "* * * * *" --runtime python3.6 --handler helloget.foo --from-file examples/helloget.py
 
 scheduled-get-python-verify:
 	number="1"; \
@@ -92,35 +88,35 @@ scheduled-get-python-verify:
 timeout-python:
 	$(eval TMPDIR := $(shell mktemp -d))
 	printf 'def foo(event, context):\n%4swhile 1: pass\n%4sreturn "hello world"\n' > $(TMPDIR)/hello-loop.py
-	kubeless function deploy timeout-python --runtime python2.7 --handler helloget.foo  --from-file $(TMPDIR)/hello-loop.py --timeout 3
+	kubeless function deploy timeout-python --runtime python3.6 --handler helloget.foo  --from-file $(TMPDIR)/hello-loop.py --timeout 3
 	rm -rf $(TMPDIR)
 
 timeout-python-verify:
-	kubectl rollout status deployment/timeout-python && sleep 2
+	$(call rollout-status,$@)
 	$(eval MSG := $(shell kubeless function call timeout-python 2>&1 || true))
 	echo $(MSG) | egrep Request.timeout.exceeded
 
 get-python-secrets:
 	kubectl create secret generic test-secret --from-literal=key=MY_KEY || true
-	kubeless function deploy get-python-secrets --runtime python2.7 --handler helloget.foo --from-file examples/helloget.py --secrets test-secret
+	kubeless function deploy get-python-secrets --runtime python3.6 --handler helloget.foo --from-file examples/helloget.py --secrets test-secret
 
 get-python-secrets-verify:
-	kubectl rollout status deployment/get-python-secrets && sleep 2
+	$(call rollout-status,$@)
 	$(eval pod := $(shell kubectl get pod -l function=get-python-secrets -o go-template -o custom-columns=:metadata.name --no-headers=true))
 	kubectl exec -it $(pod) cat /test-secret/key | egrep "MY_KEY"
 
 custom-get-python:
 	kubeless function deploy --runtime-image kubeless/get-python-example@sha256:6a14400f14e26d46a971445b7a850af533fe40cb75a67297283bdf536e09ca5e custom-get-python
 
 custom-get-python-verify:
-	kubectl rollout status deployment/custom-get-python && sleep 2
+	$(call rollout-status,$@)
 	kubeless function call custom-get-python |egrep hello.world
 
 post-python:
-	kubeless function deploy post-python --runtime python2.7 --handler hellowithdata.handler --from-file examples/hellowithdata.py
+	kubeless function deploy post-python --runtime python3.6 --handler hellowithdata.handler --from-file examples/hellowithdata.py
 
 post-python-verify:
-	kubectl rollout status deployment/post-python && sleep 2
+	$(call rollout-status,$@)
 	kubeless function call post-python --data '{"it-s": "alive"}'|egrep "it.*alive"
 	# Verify event context
 	logs=`kubectl logs --tail=1000 -l function=post-python`; \
@@ -131,9 +127,17 @@ post-python-verify:
 	echo $$logs | grep -q "event-id.*"
 
 post-python-custom-port:
-	kubeless function deploy post-python-custom-port --runtime python2.7 --handler hellowithdata.handler --from-file examples/hellowithdata.py --port 8081
+	kubeless function deploy post-python-custom-port --runtime python3.6 --handler hellowithdata.handler --from-file examples/hellowithdata.py --port 8081
 
 post-python-custom-port-verify:
-	kubectl rollout status deployment/post-python-custom-port && sleep 2
+	$(call rollout-status,$@)
 	kubectl get svc post-python-custom-port -o yaml | grep 'targetPort: 8081'
 	kubeless function call post-python-custom-port --data '{"it-s": "alive"}'|egrep "it.*alive"
+
+python-preload:
+	cd examples && zip preload.zip preload.py preloadhelper.py
+	kubeless function deploy python-preload --runtime python3.6 --handler preload.handler --from-file examples/preload.zip --timeout 3
+
+python-preload-verify:
+	$(call rollout-status,$@,10)
+	kubeless function call python-preload |egrep hello.world

stable/python/Makefile

@@ -1 +1,2 @@
 - andresmgot
+- varunbpatil