From: Gilchrist Dadaglo Date: Wed, 6 May 2020 12:25:31 +0000 (+0000) Subject: MAJOR: contrib: porting spoa_server to support python3 X-Git-Tag: v2.2-dev8~91 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3e235d38ecbfbea1f7e600b6e5281b00c9c1d294;p=thirdparty%2Fhaproxy.git MAJOR: contrib: porting spoa_server to support python3 Background: Python 2 is no longer supported since January, 1st 2020 as per https://www.python.org/doc/sunset-python-2/ The purpose of this change is to make the spoa_server contrib library compatible with Python 3 to allow transition to Python 3. Test Settings: ps_python.py: ... spoa.set_var_null("null", spoa.scope_txn) spoa.set_var_boolean("boolean", spoa.scope_txn, True) spoa.set_var_int32("int32", spoa.scope_txn, 1234) spoa.set_var_uint32("uint32", spoa.scope_txn, 1234) spoa.set_var_int64("int64", spoa.scope_txn, 1234) spoa.set_var_uint64("uint64", spoa.scope_txn, 1234) spoa.set_var_ipv4("ipv4", spoa.scope_txn, ipaddress.IPv4Address(u"127.0.0.1")) spoa.set_var_ipv6("ipv6", spoa.scope_txn, ipaddress.IPv6Address(u"1::f")) spoa.set_var_str("str", spoa.scope_txn, "1::f") spoa.set_var_bin("bin", spoa.scope_txn, "1:\x01:\x42f\x63\x63") spoa.set_var_str("python_version", spoa.scope_sess, str(sys.version_info)) ... haproxy.cfg: ... http-request capture var(txn.verb.null),debug len 1 http-request capture var(txn.verb.boolean),debug len 1 http-request capture var(txn.verb.int32),debug len 4 http-request capture var(txn.verb.uint32),debug len 4 http-request capture var(txn.verb.int64),debug len 4 http-request capture var(txn.verb.uint64),debug len 4 http-request capture var(txn.verb.ipv4),debug len 16 http-request capture var(txn.verb.ipv6),debug len 45 http-request capture var(txn.verb.str),debug len 32 http-request capture var(txn.verb.bin),debug len 32 http-request capture var(sess.verb.python_version),debug len 100 ... Test result: Python 3.8: ft_public ft_public/ 0/-1/-1/-1/0 403 212 - - PR-- 1/1/0/0/0 0/0 {|1|1234|1234|1234|1234|127.0.0.1|1::f|1::f|1:#01:Bfcc|sys.version_info(major=3, minor=8, micro=1, releaselevel='final', serial=0)} "POST / HTTP/1.1" Python 3.7: ft_public ft_public/ 0/-1/-1/-1/0 403 212 - - PR-- 1/1/0/0/0 0/0 {|1|1234|1234|1234|1234|127.0.0.1|1::f|1::f|1:#01:Bfcc|sys.version_info(major=3, minor=7, micro=6, releaselevel='final', serial=0)} "POST / HTTP/1.1" Python 3.6: ft_public ft_public/ 0/-1/-1/-1/0 403 212 - - PR-- 1/1/0/0/0 0/0 {|1|1234|1234|1234|1234|127.0.0.1|1::f|1::f|1:#01:Bfcc|sys.version_info(major=3, minor=6, micro=10, releaselevel='final', serial=0)} "POST / HTTP/1.1" Python 2.7: ft_public ft_public/ 0/-1/-1/-1/0 403 212 - - PR-- 1/1/0/0/0 0/0 {|1|1234|1234|1234|1234|127.0.0.1|1::f|1::f|1:#01:Bfcc|sys.version_info(major=2, minor=7, micro=17, releaselevel='final', serial=0)} "POST / HTTP/1.1" Not tested: Python <2.7 --- diff --git a/contrib/spoa_server/Makefile b/contrib/spoa_server/Makefile index f0752829e3..e7b20db4c0 100644 --- a/contrib/spoa_server/Makefile +++ b/contrib/spoa_server/Makefile @@ -23,10 +23,47 @@ endif ifneq ($(USE_PYTHON),) OBJS += ps_python.o + +# "--embed" flag is supported (and required) only from python 3.8+ +check_python_config := $(shell if python3-config --embed; then echo "python3.8+"; \ +elif hash python3-config; then echo "python3"; \ +elif hash python-config; then echo "python2"; fi) + +ifeq ($(check_python_config), python3.8+) +PYTHON_DEFAULT_INC := $(shell python3-config --includes) +PYTHON_DEFAULT_LIB := $(shell python3-config --libs --embed) +else ifeq ($(check_python_config), python3) +PYTHON_DEFAULT_INC := $(shell python3-config --includes) +PYTHON_DEFAULT_LIB := $(shell python3-config --libs) +else ifeq ($(check_python_config), python2) +PYTHON_DEFAULT_INC := $(shell python-config --includes) +PYTHON_DEFAULT_LIB := $(shell python-config --libs) +endif + + +# Add default path +ifneq ($(PYTHON_DEFAULT_INC),) +CFLAGS += $(PYTHON_DEFAULT_INC) +else CFLAGS += -I/usr/include/python2.7 +endif +ifneq ($(PYTHON_DEFAULT_LIB),) +LDLIBS += $(PYTHON_DEFAULT_LIB) +else LDLIBS += -lpython2.7 endif +# Add user additional paths if any +ifneq ($(PYTHON_INC),) +CFLAGS += -I$(PYTHON_INC) +endif +ifneq ($(PYTHON_LIB),) +LDLIBS += -L$(PYTHON_LIB) +endif + +LDLIBS +=-Wl,--export-dynamic +endif + spoa: $(OBJS) $(LD) $(LDFLAGS) -o $@ $^ $(LDLIBS) diff --git a/contrib/spoa_server/README b/contrib/spoa_server/README index 341f5f978d..f654a2321d 100644 --- a/contrib/spoa_server/README +++ b/contrib/spoa_server/README @@ -14,9 +14,10 @@ is done. You have to install the development packages, either from the distribution repositories or from the source. -CentOS/RHEL: yum install python-devel +CentOS/RHEL: sudo yum install python3-devel -The current python version in use is 2.7. +The current minimal python version compatible with this library is 2.7. +It's recommended to use python version 3 where possible due to python 2 deprecation. Compilation @@ -28,6 +29,11 @@ USE_LUA=1 and/or USE_PYTHON=1. You can add LUA_INC=.. LUA_LIB=.. to the make command to set the paths to the lua header files and lua libraries. +Similarly, you can add PYTHON_INC=.. PYTHON_LIB=.. to the make command to set the paths to +the python header files and python libraries. +By default, it will try to compile by detecting the default python 3 parameters. +It will fall back to python 2 if python 3 is not available. + Start the service --------------------- diff --git a/contrib/spoa_server/ps_python.c b/contrib/spoa_server/ps_python.c index 2dbe5ee386..00cb0e30db 100644 --- a/contrib/spoa_server/ps_python.c +++ b/contrib/spoa_server/ps_python.c @@ -1,13 +1,25 @@ /* spoa-server: processing Python * * Copyright 2018 OZON / Thierry Fournier + * Copyright (C) 2020 Gilchrist Dadaglo * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. * + * This program is provided in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +/* + * Define PY_SSIZE_T_CLEAN before including Python.h + * as per https://docs.python.org/3/c-api/arg.html and https://docs.python.org/2/c-api/arg.html */ +#define PY_SSIZE_T_CLEAN #include @@ -15,8 +27,10 @@ #include #include +#include #include "spoa.h" +#include "ps_python.h" /* Embedding python documentation: * @@ -43,6 +57,28 @@ static struct ps ps_python_bindings = { .ext = ".py", }; +static int ps_python_check_overflow(Py_ssize_t len) +{ + /* There might be an overflow when converting from Py_ssize_t to int. + * This function will catch those cases. + * Also, spoa "struct chunk" is limited to int size. + * We should not send data bigger than it can handle. + */ + if (len >= (Py_ssize_t)INT_MAX) { + PyErr_Format(spoa_error, + "%d is over 2GB. Please split in smaller pieces.", \ + len); + return -1; + } else { + return Py_SAFE_DOWNCAST(len, Py_ssize_t, int); + } +} + +#if IS_PYTHON_3K +static PyObject *module_spoa; +static PyObject *PyInit_spoa_module(void); +#endif /* IS_PYTHON_3K */ + static PyObject *ps_python_register_message(PyObject *self, PyObject *args) { const char *name; @@ -60,12 +96,16 @@ static PyObject *ps_python_register_message(PyObject *self, PyObject *args) static PyObject *ps_python_set_var_null(PyObject *self, PyObject *args) { const char *name; - int name_len; + Py_ssize_t name_len; + int name_len_i; int scope; if (!PyArg_ParseTuple(args, "s#i", &name, &name_len, &scope)) return NULL; - if (!set_var_null(worker, name, name_len, scope)) { + name_len_i = ps_python_check_overflow(name_len); + if (name_len_i == -1) + return NULL; + if (!set_var_null(worker, name, name_len_i, scope)) { PyErr_SetString(spoa_error, "No space left available"); return NULL; } @@ -75,13 +115,17 @@ static PyObject *ps_python_set_var_null(PyObject *self, PyObject *args) static PyObject *ps_python_set_var_boolean(PyObject *self, PyObject *args) { const char *name; - int name_len; + Py_ssize_t name_len; int scope; int value; + int name_len_i; if (!PyArg_ParseTuple(args, "s#ii", &name, &name_len, &scope, &value)) return NULL; - if (!set_var_bool(worker, name, name_len, scope, value)) { + name_len_i = ps_python_check_overflow(name_len); + if (name_len_i == -1) + return NULL; + if (!set_var_bool(worker, name, name_len_i, scope, value)) { PyErr_SetString(spoa_error, "No space left available"); return NULL; } @@ -91,13 +135,17 @@ static PyObject *ps_python_set_var_boolean(PyObject *self, PyObject *args) static PyObject *ps_python_set_var_int32(PyObject *self, PyObject *args) { const char *name; - int name_len; + Py_ssize_t name_len; int scope; int32_t value; + int name_len_i; if (!PyArg_ParseTuple(args, "s#ii", &name, &name_len, &scope, &value)) return NULL; - if (!set_var_int32(worker, name, name_len, scope, value)) { + name_len_i = ps_python_check_overflow(name_len); + if (name_len_i == -1) + return NULL; + if (!set_var_int32(worker, name, name_len_i, scope, value)) { PyErr_SetString(spoa_error, "No space left available"); return NULL; } @@ -107,13 +155,17 @@ static PyObject *ps_python_set_var_int32(PyObject *self, PyObject *args) static PyObject *ps_python_set_var_uint32(PyObject *self, PyObject *args) { const char *name; - int name_len; + Py_ssize_t name_len; int scope; uint32_t value; + int name_len_i; if (!PyArg_ParseTuple(args, "s#iI", &name, &name_len, &scope, &value)) return NULL; - if (!set_var_uint32(worker, name, name_len, scope, value)) { + name_len_i = ps_python_check_overflow(name_len); + if (name_len_i == -1) + return NULL; + if (!set_var_uint32(worker, name, name_len_i, scope, value)) { PyErr_SetString(spoa_error, "No space left available"); return NULL; } @@ -123,13 +175,17 @@ static PyObject *ps_python_set_var_uint32(PyObject *self, PyObject *args) static PyObject *ps_python_set_var_int64(PyObject *self, PyObject *args) { const char *name; - int name_len; + Py_ssize_t name_len; int scope; int64_t value; + int name_len_i; if (!PyArg_ParseTuple(args, "s#il", &name, &name_len, &scope, &value)) return NULL; - if (!set_var_int64(worker, name, name_len, scope, value)) { + name_len_i = ps_python_check_overflow(name_len); + if (name_len_i == -1) + return NULL; + if (!set_var_int64(worker, name, name_len_i, scope, value)) { PyErr_SetString(spoa_error, "No space left available"); return NULL; } @@ -139,13 +195,17 @@ static PyObject *ps_python_set_var_int64(PyObject *self, PyObject *args) static PyObject *ps_python_set_var_uint64(PyObject *self, PyObject *args) { const char *name; - int name_len; + Py_ssize_t name_len; int scope; uint64_t value; + int name_len_i; if (!PyArg_ParseTuple(args, "s#ik", &name, &name_len, &scope, &value)) return NULL; - if (!set_var_uint64(worker, name, name_len, scope, value)) { + name_len_i = ps_python_check_overflow(name_len); + if (name_len_i == -1) + return NULL; + if (!set_var_uint64(worker, name, name_len_i, scope, value)) { PyErr_SetString(spoa_error, "No space left available"); return NULL; } @@ -155,14 +215,18 @@ static PyObject *ps_python_set_var_uint64(PyObject *self, PyObject *args) static PyObject *ps_python_set_var_ipv4(PyObject *self, PyObject *args) { const char *name; - int name_len; + Py_ssize_t name_len; int scope; PyObject *ipv4; PyObject *value; struct in_addr ip; + int name_len_i; if (!PyArg_ParseTuple(args, "s#iO", &name, &name_len, &scope, &ipv4)) return NULL; + name_len_i = ps_python_check_overflow(name_len); + if (name_len_i == -1) + return NULL; if (!PyObject_IsInstance(ipv4, ipv4_address)) { PyErr_Format(spoa_error, "must be 'IPv4Address', not '%s'", ipv4->ob_type->tp_name); return NULL; @@ -171,12 +235,12 @@ static PyObject *ps_python_set_var_ipv4(PyObject *self, PyObject *args) value = PyObject_GetAttrString(ipv4, "packed"); if (value == NULL) return NULL; - if (PyString_GET_SIZE(value) != sizeof(ip)) { + if (PY_STRING_GET_SIZE(value) != sizeof(ip)) { PyErr_Format(spoa_error, "UPv6 manipulation internal error"); return NULL; } - memcpy(&ip, PyString_AS_STRING(value), PyString_GET_SIZE(value)); - if (!set_var_ipv4(worker, name, name_len, scope, &ip)) { + memcpy(&ip, PY_STRING_AS_STRING(value), PY_STRING_GET_SIZE(value)); + if (!set_var_ipv4(worker, name, name_len_i, scope, &ip)) { PyErr_SetString(spoa_error, "No space left available"); return NULL; } @@ -186,14 +250,18 @@ static PyObject *ps_python_set_var_ipv4(PyObject *self, PyObject *args) static PyObject *ps_python_set_var_ipv6(PyObject *self, PyObject *args) { const char *name; - int name_len; + Py_ssize_t name_len; int scope; PyObject *ipv6; PyObject *value; struct in6_addr ip; + int name_len_i; if (!PyArg_ParseTuple(args, "s#iO", &name, &name_len, &scope, &ipv6)) return NULL; + name_len_i = ps_python_check_overflow(name_len); + if (name_len_i == -1) + return NULL; if (!PyObject_IsInstance(ipv6, ipv6_address)) { PyErr_Format(spoa_error, "must be 'IPv6Address', not '%s'", ipv6->ob_type->tp_name); return NULL; @@ -202,12 +270,12 @@ static PyObject *ps_python_set_var_ipv6(PyObject *self, PyObject *args) value = PyObject_GetAttrString(ipv6, "packed"); if (value == NULL) return NULL; - if (PyString_GET_SIZE(value) != sizeof(ip)) { + if (PY_STRING_GET_SIZE(value) != sizeof(ip)) { PyErr_Format(spoa_error, "UPv6 manipulation internal error"); return NULL; } - memcpy(&ip, PyString_AS_STRING(value), PyString_GET_SIZE(value)); - if (!set_var_ipv6(worker, name, name_len, scope, &ip)) { + memcpy(&ip, PY_STRING_AS_STRING(value), PY_STRING_GET_SIZE(value)); + if (!set_var_ipv6(worker, name, name_len_i, scope, &ip)) { PyErr_SetString(spoa_error, "No space left available"); return NULL; } @@ -217,14 +285,20 @@ static PyObject *ps_python_set_var_ipv6(PyObject *self, PyObject *args) static PyObject *ps_python_set_var_str(PyObject *self, PyObject *args) { const char *name; - int name_len; + Py_ssize_t name_len; int scope; const char *value; - int value_len; + Py_ssize_t value_len; + int name_len_i; + int value_len_i; if (!PyArg_ParseTuple(args, "s#is#", &name, &name_len, &scope, &value, &value_len)) return NULL; - if (!set_var_string(worker, name, name_len, scope, value, value_len)) { + name_len_i = ps_python_check_overflow(name_len); + value_len_i = ps_python_check_overflow(value_len); + if (name_len_i == -1 || value_len_i == -1) + return NULL; + if (!set_var_string(worker, name, name_len_i, scope, value, value_len_i)) { PyErr_SetString(spoa_error, "No space left available"); return NULL; } @@ -234,14 +308,20 @@ static PyObject *ps_python_set_var_str(PyObject *self, PyObject *args) static PyObject *ps_python_set_var_bin(PyObject *self, PyObject *args) { const char *name; - int name_len; + Py_ssize_t name_len; int scope; const char *value; - int value_len; + Py_ssize_t value_len; + int name_len_i; + int value_len_i; if (!PyArg_ParseTuple(args, "s#is#", &name, &name_len, &scope, &value, &value_len)) return NULL; - if (!set_var_bin(worker, name, name_len, scope, value, value_len)) { + name_len_i = ps_python_check_overflow(name_len); + value_len_i = ps_python_check_overflow(value_len); + if (name_len_i == -1 || value_len_i == -1) + return NULL; + if (!set_var_bin(worker, name, name_len_i, scope, value, value_len_i)) { PyErr_SetString(spoa_error, "No space left available"); return NULL; } @@ -275,6 +355,25 @@ static PyMethodDef spoa_methods[] = { { /* end */ } }; +#if IS_PYTHON_3K +static struct PyModuleDef spoa_module_definition = { + PyModuleDef_HEAD_INIT, /* m_base */ + "spoa", /* m_name */ + "HAProxy SPOA module for python", /* m_doc */ + -1, /* m_size */ + spoa_methods, /* m_methods */ + NULL, /* m_slots */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; + +static PyObject *PyInit_spoa_module(void) +{ + return module_spoa; +} +#endif /* IS_PYTHON_3K */ + static int ps_python_start_worker(struct worker *w) { PyObject *m; @@ -282,10 +381,17 @@ static int ps_python_start_worker(struct worker *w) PyObject *value; int ret; +#if IS_PYTHON_27 Py_SetProgramName("spoa-server"); +#endif /* IS_PYTHON_27 */ +#if IS_PYTHON_3K + Py_SetProgramName(Py_DecodeLocale("spoa-server", NULL)); + PyImport_AppendInittab("spoa", &PyInit_spoa_module); +#endif /* IS_PYTHON_3K */ + Py_Initialize(); - module_name = PyString_FromString("ipaddress"); + module_name = PY_STRING_FROM_STRING("ipaddress"); if (module_name == NULL) { PyErr_Print(); return 0; @@ -310,7 +416,7 @@ static int ps_python_start_worker(struct worker *w) return 0; } - m = Py_InitModule("spoa", spoa_methods); + PY_INIT_MODULE(m, "spoa", spoa_methods, &spoa_module_definition); if (m == NULL) { PyErr_Print(); return 0; @@ -387,6 +493,9 @@ static int ps_python_start_worker(struct worker *w) return 0; } +#if IS_PYTHON_3K + module_spoa = m; +#endif /* IS_PYTHON_3K */ worker = w; return 1; } @@ -452,14 +561,14 @@ static int ps_python_exec_message(struct worker *w, void *ref, int nargs, struct /* Create the name entry */ - key = PyString_FromString("name"); + key = PY_STRING_FROM_STRING("name"); if (key == NULL) { Py_DECREF(kw_args); PyErr_Print(); return 0; } - value = PyString_FromStringAndSize(args[i].name.str, args[i].name.len); + value = PY_STRING_FROM_STRING_AND_SIZE(args[i].name.str, args[i].name.len); if (value == NULL) { Py_DECREF(kw_args); Py_DECREF(ent); @@ -480,7 +589,7 @@ static int ps_python_exec_message(struct worker *w, void *ref, int nargs, struct /* Create th value entry */ - key = PyString_FromString("value"); + key = PY_STRING_FROM_STRING("value"); if (key == NULL) { Py_DECREF(kw_args); Py_DECREF(ent); @@ -531,7 +640,7 @@ static int ps_python_exec_message(struct worker *w, void *ref, int nargs, struct PyErr_Print(); return 0; } - ip_name = PyString_FromString("address"); + ip_name = PY_STRING_FROM_STRING("address"); if (ip_name == NULL) { Py_DECREF(kw_args); Py_DECREF(ent); @@ -564,10 +673,10 @@ static int ps_python_exec_message(struct worker *w, void *ref, int nargs, struct break; case SPOE_DATA_T_STR: - value = PyString_FromStringAndSize(args[i].value.u.buffer.str, args[i].value.u.buffer.len); + value = PY_STRING_FROM_STRING_AND_SIZE(args[i].value.u.buffer.str, args[i].value.u.buffer.len); break; case SPOE_DATA_T_BIN: - value = PyString_FromStringAndSize(args[i].value.u.buffer.str, args[i].value.u.buffer.len); + value = PY_BYTES_FROM_STRING_AND_SIZE(args[i].value.u.buffer.str, args[i].value.u.buffer.len); break; default: value = Py_None; @@ -611,7 +720,7 @@ static int ps_python_exec_message(struct worker *w, void *ref, int nargs, struct return 0; } - key = PyString_FromString("args"); + key = PY_STRING_FROM_STRING("args"); if (key == NULL) { Py_DECREF(kw_args); Py_DECREF(fkw); diff --git a/contrib/spoa_server/ps_python.h b/contrib/spoa_server/ps_python.h new file mode 100644 index 0000000000..069c51dcc2 --- /dev/null +++ b/contrib/spoa_server/ps_python.h @@ -0,0 +1,52 @@ +/* ps_python.h: SPOA Python processing includes + * + * Copyright (C) 2020 Gilchrist Dadaglo + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + * + * This program is provided in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef __PS_PYTHON_H__ +#define __PS_PYTHON_H__ + +#include + +#if PY_MAJOR_VERSION >= 3 + #define IS_PYTHON_3K 1 + #define IS_PYTHON_27 0 +#elif PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION == 7 + #define IS_PYTHON_3K 0 + #define IS_PYTHON_27 1 +#else + #error "Unsupported Python Version - Please use Python 3" +#endif /* PY_MAJOR_VERSION */ + +#if IS_PYTHON_3K + #define PY_INIT_MODULE(instance, name, methods, moduledef) \ + (instance = PyModule_Create(moduledef)) + #define PY_STRING_FROM_STRING PyUnicode_FromString + #define PY_STRING_FROM_STRING_AND_SIZE PyUnicode_FromStringAndSize + #define PY_BYTES_FROM_STRING_AND_SIZE PyBytes_FromStringAndSize + #define PY_STRING_GET_SIZE PyBytes_Size + #define PY_STRING_AS_STRING PyBytes_AsString +#elif IS_PYTHON_27 + #define PY_INIT_MODULE(instance, name, methods, moduledef) \ + (instance = Py_InitModule(name, methods)) + #define PY_STRING_FROM_STRING PyString_FromString + #define PY_STRING_FROM_STRING_AND_SIZE PyString_FromStringAndSize + #define PY_BYTES_FROM_STRING_AND_SIZE PyString_FromStringAndSize + #define PY_STRING_GET_SIZE PyString_GET_SIZE + #define PY_STRING_AS_STRING PyString_AS_STRING +#endif /* IS_PYTHON_3K */ + +#endif /* __PS_PYTHON_H__ */ + +/* EOF */