MAJOR: contrib: porting spoa_server to support python3
Background:
Python 2 is no longer supported since January, 1st 2020 as per
https://www.python.org/doc/sunset-python-2/
The purpose of this change is to make the spoa_server contrib library
compatible with Python 3 to allow transition to Python 3.
Test Settings:
ps_python.py:
...
spoa.set_var_null("null", spoa.scope_txn)
spoa.set_var_boolean("boolean", spoa.scope_txn, True)
spoa.set_var_int32("int32", spoa.scope_txn, 1234)
spoa.set_var_uint32("uint32", spoa.scope_txn, 1234)
spoa.set_var_int64("int64", spoa.scope_txn, 1234)
spoa.set_var_uint64("uint64", spoa.scope_txn, 1234)
spoa.set_var_ipv4("ipv4", spoa.scope_txn, ipaddress.IPv4Address(u"127.0.0.1"))
spoa.set_var_ipv6("ipv6", spoa.scope_txn, ipaddress.IPv6Address(u"1::f"))
spoa.set_var_str("str", spoa.scope_txn, "1::f")
spoa.set_var_bin("bin", spoa.scope_txn, "1:\x01:\x42f\x63\x63")
spoa.set_var_str("python_version", spoa.scope_sess, str(sys.version_info))
...
haproxy.cfg:
...
http-request capture var(txn.verb.null),debug len 1
http-request capture var(txn.verb.boolean),debug len 1
http-request capture var(txn.verb.int32),debug len 4
http-request capture var(txn.verb.uint32),debug len 4
http-request capture var(txn.verb.int64),debug len 4
http-request capture var(txn.verb.uint64),debug len 4
http-request capture var(txn.verb.ipv4),debug len 16
http-request capture var(txn.verb.ipv6),debug len 45
http-request capture var(txn.verb.str),debug len 32
http-request capture var(txn.verb.bin),debug len 32
http-request capture var(sess.verb.python_version),debug len 100
...
Test result:
Python 3.8:
ft_public ft_public/<NOSRV> 0/-1/-1/-1/0 403 212 - - PR-- 1/1/0/0/0 0/0 {|1|1234|1234|1234|1234|127.0.0.1|1::f|1::f|1:#01:Bfcc|sys.version_info(major=3, minor=8, micro=1, releaselevel='final', serial=0)} "POST / HTTP/1.1"
Python 3.7:
ft_public ft_public/<NOSRV> 0/-1/-1/-1/0 403 212 - - PR-- 1/1/0/0/0 0/0 {|1|1234|1234|1234|1234|127.0.0.1|1::f|1::f|1:#01:Bfcc|sys.version_info(major=3, minor=7, micro=6, releaselevel='final', serial=0)} "POST / HTTP/1.1"
Python 3.6:
ft_public ft_public/<NOSRV> 0/-1/-1/-1/0 403 212 - - PR-- 1/1/0/0/0 0/0 {|1|1234|1234|1234|1234|127.0.0.1|1::f|1::f|1:#01:Bfcc|sys.version_info(major=3, minor=6, micro=10, releaselevel='final', serial=0)} "POST / HTTP/1.1"
Python 2.7:
ft_public ft_public/<NOSRV> 0/-1/-1/-1/0 403 212 - - PR-- 1/1/0/0/0 0/0 {|1|1234|1234|1234|1234|127.0.0.1|1::f|1::f|1:#01:Bfcc|sys.version_info(major=2, minor=7, micro=17, releaselevel='final', serial=0)} "POST / HTTP/1.1"
Not tested:
Python <2.7
diff --git a/contrib/spoa_server/Makefile b/contrib/spoa_server/Makefile
index f075282..e7b20db 100644
--- a/contrib/spoa_server/Makefile
+++ b/contrib/spoa_server/Makefile
@@ -23,10 +23,47 @@
ifneq ($(USE_PYTHON),)
OBJS += ps_python.o
+
+# "--embed" flag is supported (and required) only from python 3.8+
+check_python_config := $(shell if python3-config --embed; then echo "python3.8+"; \
+elif hash python3-config; then echo "python3"; \
+elif hash python-config; then echo "python2"; fi)
+
+ifeq ($(check_python_config), python3.8+)
+PYTHON_DEFAULT_INC := $(shell python3-config --includes)
+PYTHON_DEFAULT_LIB := $(shell python3-config --libs --embed)
+else ifeq ($(check_python_config), python3)
+PYTHON_DEFAULT_INC := $(shell python3-config --includes)
+PYTHON_DEFAULT_LIB := $(shell python3-config --libs)
+else ifeq ($(check_python_config), python2)
+PYTHON_DEFAULT_INC := $(shell python-config --includes)
+PYTHON_DEFAULT_LIB := $(shell python-config --libs)
+endif
+
+
+# Add default path
+ifneq ($(PYTHON_DEFAULT_INC),)
+CFLAGS += $(PYTHON_DEFAULT_INC)
+else
CFLAGS += -I/usr/include/python2.7
+endif
+ifneq ($(PYTHON_DEFAULT_LIB),)
+LDLIBS += $(PYTHON_DEFAULT_LIB)
+else
LDLIBS += -lpython2.7
endif
+# Add user additional paths if any
+ifneq ($(PYTHON_INC),)
+CFLAGS += -I$(PYTHON_INC)
+endif
+ifneq ($(PYTHON_LIB),)
+LDLIBS += -L$(PYTHON_LIB)
+endif
+
+LDLIBS +=-Wl,--export-dynamic
+endif
+
spoa: $(OBJS)
$(LD) $(LDFLAGS) -o $@ $^ $(LDLIBS)
diff --git a/contrib/spoa_server/README b/contrib/spoa_server/README
index 341f5f9..f654a23 100644
--- a/contrib/spoa_server/README
+++ b/contrib/spoa_server/README
@@ -14,9 +14,10 @@
You have to install the development packages, either from the
distribution repositories or from the source.
-CentOS/RHEL: yum install python-devel
+CentOS/RHEL: sudo yum install python3-devel
-The current python version in use is 2.7.
+The current minimal python version compatible with this library is 2.7.
+It's recommended to use python version 3 where possible due to python 2 deprecation.
Compilation
@@ -28,6 +29,11 @@
You can add LUA_INC=.. LUA_LIB=.. to the make command to set the paths to
the lua header files and lua libraries.
+Similarly, you can add PYTHON_INC=.. PYTHON_LIB=.. to the make command to set the paths to
+the python header files and python libraries.
+By default, it will try to compile by detecting the default python 3 parameters.
+It will fall back to python 2 if python 3 is not available.
+
Start the service
---------------------
diff --git a/contrib/spoa_server/ps_python.c b/contrib/spoa_server/ps_python.c
index 2dbe5ee..00cb0e3 100644
--- a/contrib/spoa_server/ps_python.c
+++ b/contrib/spoa_server/ps_python.c
@@ -1,13 +1,25 @@
/* spoa-server: processing Python
*
* Copyright 2018 OZON / Thierry Fournier <thierry.fournier@ozon.io>
+ * Copyright (C) 2020 Gilchrist Dadaglo <gilchrist@dadaglo.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*
+ * This program is provided in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+/*
+ * Define PY_SSIZE_T_CLEAN before including Python.h
+ * as per https://docs.python.org/3/c-api/arg.html and https://docs.python.org/2/c-api/arg.html
*/
+#define PY_SSIZE_T_CLEAN
#include <Python.h>
@@ -15,8 +27,10 @@
#include <errno.h>
#include <string.h>
+#include <limits.h>
#include "spoa.h"
+#include "ps_python.h"
/* Embedding python documentation:
*
@@ -43,6 +57,28 @@
.ext = ".py",
};
+static int ps_python_check_overflow(Py_ssize_t len)
+{
+ /* There might be an overflow when converting from Py_ssize_t to int.
+ * This function will catch those cases.
+ * Also, spoa "struct chunk" is limited to int size.
+ * We should not send data bigger than it can handle.
+ */
+ if (len >= (Py_ssize_t)INT_MAX) {
+ PyErr_Format(spoa_error,
+ "%d is over 2GB. Please split in smaller pieces.", \
+ len);
+ return -1;
+ } else {
+ return Py_SAFE_DOWNCAST(len, Py_ssize_t, int);
+ }
+}
+
+#if IS_PYTHON_3K
+static PyObject *module_spoa;
+static PyObject *PyInit_spoa_module(void);
+#endif /* IS_PYTHON_3K */
+
static PyObject *ps_python_register_message(PyObject *self, PyObject *args)
{
const char *name;
@@ -60,12 +96,16 @@
static PyObject *ps_python_set_var_null(PyObject *self, PyObject *args)
{
const char *name;
- int name_len;
+ Py_ssize_t name_len;
+ int name_len_i;
int scope;
if (!PyArg_ParseTuple(args, "s#i", &name, &name_len, &scope))
return NULL;
+ name_len_i = ps_python_check_overflow(name_len);
+ if (name_len_i == -1)
+ return NULL;
- if (!set_var_null(worker, name, name_len, scope)) {
+ if (!set_var_null(worker, name, name_len_i, scope)) {
PyErr_SetString(spoa_error, "No space left available");
return NULL;
}
@@ -75,13 +115,17 @@
static PyObject *ps_python_set_var_boolean(PyObject *self, PyObject *args)
{
const char *name;
- int name_len;
+ Py_ssize_t name_len;
int scope;
int value;
+ int name_len_i;
if (!PyArg_ParseTuple(args, "s#ii", &name, &name_len, &scope, &value))
return NULL;
- if (!set_var_bool(worker, name, name_len, scope, value)) {
+ name_len_i = ps_python_check_overflow(name_len);
+ if (name_len_i == -1)
+ return NULL;
+ if (!set_var_bool(worker, name, name_len_i, scope, value)) {
PyErr_SetString(spoa_error, "No space left available");
return NULL;
}
@@ -91,13 +135,17 @@
static PyObject *ps_python_set_var_int32(PyObject *self, PyObject *args)
{
const char *name;
- int name_len;
+ Py_ssize_t name_len;
int scope;
int32_t value;
+ int name_len_i;
if (!PyArg_ParseTuple(args, "s#ii", &name, &name_len, &scope, &value))
return NULL;
- if (!set_var_int32(worker, name, name_len, scope, value)) {
+ name_len_i = ps_python_check_overflow(name_len);
+ if (name_len_i == -1)
+ return NULL;
+ if (!set_var_int32(worker, name, name_len_i, scope, value)) {
PyErr_SetString(spoa_error, "No space left available");
return NULL;
}
@@ -107,13 +155,17 @@
static PyObject *ps_python_set_var_uint32(PyObject *self, PyObject *args)
{
const char *name;
- int name_len;
+ Py_ssize_t name_len;
int scope;
uint32_t value;
+ int name_len_i;
if (!PyArg_ParseTuple(args, "s#iI", &name, &name_len, &scope, &value))
return NULL;
+ name_len_i = ps_python_check_overflow(name_len);
+ if (name_len_i == -1)
+ return NULL;
- if (!set_var_uint32(worker, name, name_len, scope, value)) {
+ if (!set_var_uint32(worker, name, name_len_i, scope, value)) {
PyErr_SetString(spoa_error, "No space left available");
return NULL;
}
@@ -123,13 +175,17 @@
static PyObject *ps_python_set_var_int64(PyObject *self, PyObject *args)
{
const char *name;
- int name_len;
+ Py_ssize_t name_len;
int scope;
int64_t value;
+ int name_len_i;
if (!PyArg_ParseTuple(args, "s#il", &name, &name_len, &scope, &value))
return NULL;
- if (!set_var_int64(worker, name, name_len, scope, value)) {
+ name_len_i = ps_python_check_overflow(name_len);
+ if (name_len_i == -1)
+ return NULL;
+ if (!set_var_int64(worker, name, name_len_i, scope, value)) {
PyErr_SetString(spoa_error, "No space left available");
return NULL;
}
@@ -139,13 +195,17 @@
static PyObject *ps_python_set_var_uint64(PyObject *self, PyObject *args)
{
const char *name;
- int name_len;
+ Py_ssize_t name_len;
int scope;
uint64_t value;
+ int name_len_i;
if (!PyArg_ParseTuple(args, "s#ik", &name, &name_len, &scope, &value))
return NULL;
- if (!set_var_uint64(worker, name, name_len, scope, value)) {
+ name_len_i = ps_python_check_overflow(name_len);
+ if (name_len_i == -1)
+ return NULL;
+ if (!set_var_uint64(worker, name, name_len_i, scope, value)) {
PyErr_SetString(spoa_error, "No space left available");
return NULL;
}
@@ -155,14 +215,18 @@
static PyObject *ps_python_set_var_ipv4(PyObject *self, PyObject *args)
{
const char *name;
- int name_len;
+ Py_ssize_t name_len;
int scope;
PyObject *ipv4;
PyObject *value;
struct in_addr ip;
+ int name_len_i;
if (!PyArg_ParseTuple(args, "s#iO", &name, &name_len, &scope, &ipv4))
return NULL;
+ name_len_i = ps_python_check_overflow(name_len);
+ if (name_len_i == -1)
+ return NULL;
if (!PyObject_IsInstance(ipv4, ipv4_address)) {
PyErr_Format(spoa_error, "must be 'IPv4Address', not '%s'", ipv4->ob_type->tp_name);
return NULL;
@@ -171,12 +235,12 @@
value = PyObject_GetAttrString(ipv4, "packed");
if (value == NULL)
return NULL;
- if (PyString_GET_SIZE(value) != sizeof(ip)) {
+ if (PY_STRING_GET_SIZE(value) != sizeof(ip)) {
PyErr_Format(spoa_error, "UPv6 manipulation internal error");
return NULL;
}
- memcpy(&ip, PyString_AS_STRING(value), PyString_GET_SIZE(value));
- if (!set_var_ipv4(worker, name, name_len, scope, &ip)) {
+ memcpy(&ip, PY_STRING_AS_STRING(value), PY_STRING_GET_SIZE(value));
+ if (!set_var_ipv4(worker, name, name_len_i, scope, &ip)) {
PyErr_SetString(spoa_error, "No space left available");
return NULL;
}
@@ -186,14 +250,18 @@
static PyObject *ps_python_set_var_ipv6(PyObject *self, PyObject *args)
{
const char *name;
- int name_len;
+ Py_ssize_t name_len;
int scope;
PyObject *ipv6;
PyObject *value;
struct in6_addr ip;
+ int name_len_i;
if (!PyArg_ParseTuple(args, "s#iO", &name, &name_len, &scope, &ipv6))
return NULL;
+ name_len_i = ps_python_check_overflow(name_len);
+ if (name_len_i == -1)
+ return NULL;
if (!PyObject_IsInstance(ipv6, ipv6_address)) {
PyErr_Format(spoa_error, "must be 'IPv6Address', not '%s'", ipv6->ob_type->tp_name);
return NULL;
@@ -202,12 +270,12 @@
value = PyObject_GetAttrString(ipv6, "packed");
if (value == NULL)
return NULL;
- if (PyString_GET_SIZE(value) != sizeof(ip)) {
+ if (PY_STRING_GET_SIZE(value) != sizeof(ip)) {
PyErr_Format(spoa_error, "UPv6 manipulation internal error");
return NULL;
}
- memcpy(&ip, PyString_AS_STRING(value), PyString_GET_SIZE(value));
- if (!set_var_ipv6(worker, name, name_len, scope, &ip)) {
+ memcpy(&ip, PY_STRING_AS_STRING(value), PY_STRING_GET_SIZE(value));
+ if (!set_var_ipv6(worker, name, name_len_i, scope, &ip)) {
PyErr_SetString(spoa_error, "No space left available");
return NULL;
}
@@ -217,14 +285,20 @@
static PyObject *ps_python_set_var_str(PyObject *self, PyObject *args)
{
const char *name;
- int name_len;
+ Py_ssize_t name_len;
int scope;
const char *value;
- int value_len;
+ Py_ssize_t value_len;
+ int name_len_i;
+ int value_len_i;
if (!PyArg_ParseTuple(args, "s#is#", &name, &name_len, &scope, &value, &value_len))
return NULL;
- if (!set_var_string(worker, name, name_len, scope, value, value_len)) {
+ name_len_i = ps_python_check_overflow(name_len);
+ value_len_i = ps_python_check_overflow(value_len);
+ if (name_len_i == -1 || value_len_i == -1)
+ return NULL;
+ if (!set_var_string(worker, name, name_len_i, scope, value, value_len_i)) {
PyErr_SetString(spoa_error, "No space left available");
return NULL;
}
@@ -234,14 +308,20 @@
static PyObject *ps_python_set_var_bin(PyObject *self, PyObject *args)
{
const char *name;
- int name_len;
+ Py_ssize_t name_len;
int scope;
const char *value;
- int value_len;
+ Py_ssize_t value_len;
+ int name_len_i;
+ int value_len_i;
if (!PyArg_ParseTuple(args, "s#is#", &name, &name_len, &scope, &value, &value_len))
return NULL;
+ name_len_i = ps_python_check_overflow(name_len);
+ value_len_i = ps_python_check_overflow(value_len);
+ if (name_len_i == -1 || value_len_i == -1)
+ return NULL;
- if (!set_var_bin(worker, name, name_len, scope, value, value_len)) {
+ if (!set_var_bin(worker, name, name_len_i, scope, value, value_len_i)) {
PyErr_SetString(spoa_error, "No space left available");
return NULL;
}
@@ -275,6 +355,25 @@
{ /* end */ }
};
+#if IS_PYTHON_3K
+static struct PyModuleDef spoa_module_definition = {
+ PyModuleDef_HEAD_INIT, /* m_base */
+ "spoa", /* m_name */
+ "HAProxy SPOA module for python", /* m_doc */
+ -1, /* m_size */
+ spoa_methods, /* m_methods */
+ NULL, /* m_slots */
+ NULL, /* m_traverse */
+ NULL, /* m_clear */
+ NULL /* m_free */
+};
+
+static PyObject *PyInit_spoa_module(void)
+{
+ return module_spoa;
+}
+#endif /* IS_PYTHON_3K */
+
static int ps_python_start_worker(struct worker *w)
{
PyObject *m;
@@ -282,10 +381,17 @@
PyObject *value;
int ret;
+#if IS_PYTHON_27
Py_SetProgramName("spoa-server");
+#endif /* IS_PYTHON_27 */
+#if IS_PYTHON_3K
+ Py_SetProgramName(Py_DecodeLocale("spoa-server", NULL));
+ PyImport_AppendInittab("spoa", &PyInit_spoa_module);
+#endif /* IS_PYTHON_3K */
+
Py_Initialize();
- module_name = PyString_FromString("ipaddress");
+ module_name = PY_STRING_FROM_STRING("ipaddress");
if (module_name == NULL) {
PyErr_Print();
return 0;
@@ -310,7 +416,7 @@
return 0;
}
- m = Py_InitModule("spoa", spoa_methods);
+ PY_INIT_MODULE(m, "spoa", spoa_methods, &spoa_module_definition);
if (m == NULL) {
PyErr_Print();
return 0;
@@ -387,6 +493,9 @@
return 0;
}
+#if IS_PYTHON_3K
+ module_spoa = m;
+#endif /* IS_PYTHON_3K */
worker = w;
return 1;
}
@@ -452,14 +561,14 @@
/* Create the name entry */
- key = PyString_FromString("name");
+ key = PY_STRING_FROM_STRING("name");
if (key == NULL) {
Py_DECREF(kw_args);
PyErr_Print();
return 0;
}
- value = PyString_FromStringAndSize(args[i].name.str, args[i].name.len);
+ value = PY_STRING_FROM_STRING_AND_SIZE(args[i].name.str, args[i].name.len);
if (value == NULL) {
Py_DECREF(kw_args);
Py_DECREF(ent);
@@ -480,7 +589,7 @@
/* Create th value entry */
- key = PyString_FromString("value");
+ key = PY_STRING_FROM_STRING("value");
if (key == NULL) {
Py_DECREF(kw_args);
Py_DECREF(ent);
@@ -531,7 +640,7 @@
PyErr_Print();
return 0;
}
- ip_name = PyString_FromString("address");
+ ip_name = PY_STRING_FROM_STRING("address");
if (ip_name == NULL) {
Py_DECREF(kw_args);
Py_DECREF(ent);
@@ -564,10 +673,10 @@
break;
case SPOE_DATA_T_STR:
- value = PyString_FromStringAndSize(args[i].value.u.buffer.str, args[i].value.u.buffer.len);
+ value = PY_STRING_FROM_STRING_AND_SIZE(args[i].value.u.buffer.str, args[i].value.u.buffer.len);
break;
case SPOE_DATA_T_BIN:
- value = PyString_FromStringAndSize(args[i].value.u.buffer.str, args[i].value.u.buffer.len);
+ value = PY_BYTES_FROM_STRING_AND_SIZE(args[i].value.u.buffer.str, args[i].value.u.buffer.len);
break;
default:
value = Py_None;
@@ -611,7 +720,7 @@
return 0;
}
- key = PyString_FromString("args");
+ key = PY_STRING_FROM_STRING("args");
if (key == NULL) {
Py_DECREF(kw_args);
Py_DECREF(fkw);
diff --git a/contrib/spoa_server/ps_python.h b/contrib/spoa_server/ps_python.h
new file mode 100644
index 0000000..069c51d
--- /dev/null
+++ b/contrib/spoa_server/ps_python.h
@@ -0,0 +1,52 @@
+/* ps_python.h: SPOA Python processing includes
+ *
+ * Copyright (C) 2020 Gilchrist Dadaglo <gilchrist@dadaglo.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * This program is provided in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef __PS_PYTHON_H__
+#define __PS_PYTHON_H__
+
+#include <Python.h>
+
+#if PY_MAJOR_VERSION >= 3
+ #define IS_PYTHON_3K 1
+ #define IS_PYTHON_27 0
+#elif PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION == 7
+ #define IS_PYTHON_3K 0
+ #define IS_PYTHON_27 1
+#else
+ #error "Unsupported Python Version - Please use Python 3"
+#endif /* PY_MAJOR_VERSION */
+
+#if IS_PYTHON_3K
+ #define PY_INIT_MODULE(instance, name, methods, moduledef) \
+ (instance = PyModule_Create(moduledef))
+ #define PY_STRING_FROM_STRING PyUnicode_FromString
+ #define PY_STRING_FROM_STRING_AND_SIZE PyUnicode_FromStringAndSize
+ #define PY_BYTES_FROM_STRING_AND_SIZE PyBytes_FromStringAndSize
+ #define PY_STRING_GET_SIZE PyBytes_Size
+ #define PY_STRING_AS_STRING PyBytes_AsString
+#elif IS_PYTHON_27
+ #define PY_INIT_MODULE(instance, name, methods, moduledef) \
+ (instance = Py_InitModule(name, methods))
+ #define PY_STRING_FROM_STRING PyString_FromString
+ #define PY_STRING_FROM_STRING_AND_SIZE PyString_FromStringAndSize
+ #define PY_BYTES_FROM_STRING_AND_SIZE PyString_FromStringAndSize
+ #define PY_STRING_GET_SIZE PyString_GET_SIZE
+ #define PY_STRING_AS_STRING PyString_AS_STRING
+#endif /* IS_PYTHON_3K */
+
+#endif /* __PS_PYTHON_H__ */
+
+/* EOF */