Skip to content

gh-103583: Add codecs and maps to _codecs_* module state#103540

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
erlend-aasland merged 5 commits into from
Apr 17, 2023
Merged

gh-103583: Add codecs and maps to _codecs_* module state #103540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ed80fe3
Rename module state struct
erlend-aasland Apr 13, 2023
f5b906f
Add codec and mapping lists to codec module states
erlend-aasland Apr 13, 2023
9973a7d
Silence unused variable warnings
erlend-aasland Apr 14, 2023
605c4fb
Pull in main
erlend-aasland Apr 16, 2023
65af110
Hide ugly code in macro
erlend-aasland Apr 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Modules/cjkcodecs/_codecs_cn.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -453,14 +453,14 @@ DECODER(hz)
}


BEGIN_MAPPINGS_LIST
BEGIN_MAPPINGS_LIST(4)
MAPPING_DECONLY(gb2312)
MAPPING_DECONLY(gbkext)
MAPPING_ENCONLY(gbcommon)
MAPPING_ENCDEC(gb18030ext)
END_MAPPINGS_LIST

BEGIN_CODECS_LIST
BEGIN_CODECS_LIST(4)
CODEC_STATELESS(gb2312)
CODEC_STATELESS(gbk)
CODEC_STATELESS(gb18030)
Expand Down
5 changes: 2 additions & 3 deletions Modules/cjkcodecs/_codecs_hk.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -177,14 +177,13 @@ DECODER(big5hkscs)
return 0;
}


BEGIN_MAPPINGS_LIST
BEGIN_MAPPINGS_LIST(3)
MAPPING_DECONLY(big5hkscs)
MAPPING_ENCONLY(big5hkscs_bmp)
MAPPING_ENCONLY(big5hkscs_nonbmp)
END_MAPPINGS_LIST

BEGIN_CODECS_LIST
BEGIN_CODECS_LIST(1)
CODEC_STATELESS_WINIT(big5hkscs)
END_CODECS_LIST

Expand Down
9 changes: 5 additions & 4 deletions Modules/cjkcodecs/_codecs_iso2022.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1119,18 +1119,19 @@ static const struct iso2022_designation iso2022_jp_ext_designations[] ={
CONFIGDEF(jp_ext, NO_SHIFT | USE_JISX0208_EXT)


BEGIN_MAPPINGS_LIST
BEGIN_MAPPINGS_LIST(0)
/* no mapping table here */
END_MAPPINGS_LIST

#define ISO2022_CODEC(variation){\
#define ISO2022_CODEC(variation) \
NEXT_CODEC = (MultibyteCodec){\
"iso2022_" #variation, \
&iso2022_##variation##_config, \
iso2022_codec_init, \
_STATEFUL_METHODS(iso2022) \
},
};

BEGIN_CODECS_LIST
BEGIN_CODECS_LIST(7)
ISO2022_CODEC(kr)
ISO2022_CODEC(jp)
ISO2022_CODEC(jp_1)
Expand Down
13 changes: 9 additions & 4 deletions Modules/cjkcodecs/_codecs_jp.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -733,7 +733,7 @@ DECODER(shift_jis_2004)
}


BEGIN_MAPPINGS_LIST
BEGIN_MAPPINGS_LIST(11)
MAPPING_DECONLY(jisx0208)
MAPPING_DECONLY(jisx0212)
MAPPING_ENCONLY(jisxcommon)
Expand All@@ -747,14 +747,19 @@ BEGIN_MAPPINGS_LIST
MAPPING_ENCDEC(cp932ext)
END_MAPPINGS_LIST

BEGIN_CODECS_LIST
#define CODEC_CUSTOM(NAME, N, METH) \
NEXT_CODEC = (MultibyteCodec){NAME, (void *)N, NULL, _STATELESS_METHODS(METH)};

BEGIN_CODECS_LIST(7)
CODEC_STATELESS(shift_jis)
CODEC_STATELESS(cp932)
CODEC_STATELESS(euc_jp)
CODEC_STATELESS(shift_jis_2004)
CODEC_STATELESS(euc_jis_2004)
{"euc_jisx0213", (void *)2000, NULL, _STATELESS_METHODS(euc_jis_2004) },
{"shift_jisx0213", (void *)2000, NULL, _STATELESS_METHODS(shift_jis_2004) },
CODEC_CUSTOM("euc_jisx0213", 2000, euc_jis_2004)
CODEC_CUSTOM("shift_jisx0213", 2000, shift_jis_2004)
END_CODECS_LIST

#undef CODEC_CUSTOM

I_AM_A_MODULE_FOR(jp)
4 changes: 2 additions & 2 deletions Modules/cjkcodecs/_codecs_kr.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -453,13 +453,13 @@ DECODER(johab)
#undef FILL


BEGIN_MAPPINGS_LIST
BEGIN_MAPPINGS_LIST(3)
MAPPING_DECONLY(ksx1001)
MAPPING_ENCONLY(cp949)
MAPPING_DECONLY(cp949ext)
END_MAPPINGS_LIST

BEGIN_CODECS_LIST
BEGIN_CODECS_LIST(3)
CODEC_STATELESS(euc_kr)
CODEC_STATELESS(cp949)
CODEC_STATELESS(johab)
Expand Down
4 changes: 2 additions & 2 deletions Modules/cjkcodecs/_codecs_tw.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -130,12 +130,12 @@ DECODER(cp950)



BEGIN_MAPPINGS_LIST
BEGIN_MAPPINGS_LIST(2)
MAPPING_ENCDEC(big5)
MAPPING_ENCDEC(cp950ext)
END_MAPPINGS_LIST

BEGIN_CODECS_LIST
BEGIN_CODECS_LIST(2)
CODEC_STATELESS(big5)
CODEC_STATELESS(cp950)
END_CODECS_LIST
Expand Down
175 changes: 118 additions & 57 deletions Modules/cjkcodecs/cjkcodecs.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -60,8 +60,20 @@ struct pair_encodemap{
DBCHAR code;
};

static const MultibyteCodec *codec_list;
static const struct dbcs_map *mapping_list;
typedef struct{
int num_mappings;
int num_codecs;
struct dbcs_map *mapping_list;
MultibyteCodec *codec_list;
} cjkcodecs_module_state;

static inline cjkcodecs_module_state *
get_module_state(PyObject *mod)
{
void *state = PyModule_GetState(mod);
assert(state != NULL);
return (cjkcodecs_module_state *)state;
}

#define CODEC_INIT(encoding) \
static int encoding##_codec_init(const void *config)
Expand DownExpand Up@@ -202,16 +214,42 @@ static const struct dbcs_map *mapping_list;
#define TRYMAP_DEC(charset, assi, c1, c2) \
_TRYMAP_DEC(&charset##_decmap[c1], assi, c2)

#define BEGIN_MAPPINGS_LIST static const struct dbcs_map _mapping_list[] ={
#define MAPPING_ENCONLY(enc){#enc, (void*)enc##_encmap, NULL},
#define MAPPING_DECONLY(enc){#enc, NULL, (void*)enc##_decmap},
#define MAPPING_ENCDEC(enc){#enc, (void*)enc##_encmap, (void*)enc##_decmap},
#define END_MAPPINGS_LIST \
{"", NULL, NULL} }; \
static const struct dbcs_map *mapping_list = \
(const struct dbcs_map *)_mapping_list;
#define BEGIN_MAPPINGS_LIST(NUM) \
static int \
add_mappings(cjkcodecs_module_state *st) \
{\
int idx = 0; \
(void)idx; \
st->num_mappings = NUM; \
st->mapping_list = PyMem_Calloc(NUM, sizeof(struct dbcs_map)); \
if (st->mapping_list == NULL){\
return -1; \
}

#define MAPPING_ENCONLY(enc) \
st->mapping_list[idx++] = (struct dbcs_map){#enc, (void*)enc##_encmap, NULL};
#define MAPPING_DECONLY(enc) \
st->mapping_list[idx++] = (struct dbcs_map){#enc, NULL, (void*)enc##_decmap};
#define MAPPING_ENCDEC(enc) \
st->mapping_list[idx++] = (struct dbcs_map){#enc, (void*)enc##_encmap, (void*)enc##_decmap};

#define END_MAPPINGS_LIST \
assert(st->num_mappings == idx); \
return 0; \
}

#define BEGIN_CODECS_LIST(NUM) \
static int \
add_codecs(cjkcodecs_module_state *st) \
{\
int idx = 0; \
(void)idx; \
st->num_codecs = NUM; \
st->codec_list = PyMem_Calloc(NUM, sizeof(MultibyteCodec)); \
if (st->codec_list == NULL){\
return -1; \
}

#define BEGIN_CODECS_LIST static const MultibyteCodec _codec_list[] ={
#define _STATEFUL_METHODS(enc) \
enc##_encode, \
enc##_encode_init, \
Expand All@@ -222,23 +260,21 @@ static const struct dbcs_map *mapping_list;
#define _STATELESS_METHODS(enc) \
enc##_encode, NULL, NULL, \
enc##_decode, NULL, NULL,
#define CODEC_STATEFUL(enc){\
#enc, NULL, NULL, \
_STATEFUL_METHODS(enc) \
},
#define CODEC_STATELESS(enc){\
#enc, NULL, NULL, \
_STATELESS_METHODS(enc) \
},
#define CODEC_STATELESS_WINIT(enc){\
#enc, NULL, \
enc##_codec_init, \
_STATELESS_METHODS(enc) \
},
#define END_CODECS_LIST \
{"", NULL,} }; \
static const MultibyteCodec *codec_list = \
(const MultibyteCodec *)_codec_list;

#define NEXT_CODEC \
st->codec_list[idx++]

#define CODEC_STATEFUL(enc) \
Copy link
Member

@corona10corona10Apr 16, 2023•
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow, readability will be frustrating (it's not your fault), I will think about a better way.. :(

Copy link
ContributorAuthor

@erlend-aaslanderlend-aaslandApr 16, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know, readability is already frustrating with these modules. If we are to keep the diff down, we just have to go with the macro template style already used. An alternative is to refactor the modules first, but that bears a risk.

NEXT_CODEC = (MultibyteCodec){#enc, NULL, NULL, _STATEFUL_METHODS(enc)};
#define CODEC_STATELESS(enc) \
NEXT_CODEC = (MultibyteCodec){#enc, NULL, NULL, _STATELESS_METHODS(enc)};
#define CODEC_STATELESS_WINIT(enc) \
NEXT_CODEC = (MultibyteCodec){#enc, NULL, enc##_codec_init, _STATELESS_METHODS(enc)};

#define END_CODECS_LIST \
assert(st->num_codecs == idx); \
return 0; \
}



Expand All@@ -249,53 +285,70 @@ getmultibytecodec(void)
}

static PyObject *
getcodec(PyObject *self, PyObject *encoding)
_getcodec(const MultibyteCodec *codec)
{
PyObject *codecobj, *r, *cofunc;
const MultibyteCodec *codec;
const char *enc;

if (!PyUnicode_Check(encoding)){
PyErr_SetString(PyExc_TypeError,
"encoding name must be a string.");
PyObject *cofunc = getmultibytecodec();
if (cofunc == NULL){
return NULL;
}
enc = PyUnicode_AsUTF8(encoding);
if (enc == NULL)
return NULL;

cofunc = getmultibytecodec();
if (cofunc == NULL)
PyObject *codecobj = PyCapsule_New((void *)codec,
PyMultibyteCodec_CAPSULE_NAME,
NULL);
if (codecobj == NULL){
Py_DECREF(cofunc);
return NULL;
}

for (codec = codec_list; codec->encoding[0]; codec++)
if (strcmp(codec->encoding, enc) == 0)
break;
PyObject *res = PyObject_CallOneArg(cofunc, codecobj);
Py_DECREF(codecobj);
Py_DECREF(cofunc);
return res;
}

if (codec->encoding[0] == '\0'){
PyErr_SetString(PyExc_LookupError,
"no such codec is supported.");
static PyObject *
getcodec(PyObject *self, PyObject *encoding)
{
if (!PyUnicode_Check(encoding)){
PyErr_SetString(PyExc_TypeError,
"encoding name must be a string.");
return NULL;
}

codecobj = PyCapsule_New((void *)codec, PyMultibyteCodec_CAPSULE_NAME, NULL);
if (codecobj == NULL)
const char *enc = PyUnicode_AsUTF8(encoding);
if (enc == NULL){
return NULL;
}

r = PyObject_CallOneArg(cofunc, codecobj);
Py_DECREF(codecobj);
Py_DECREF(cofunc);
cjkcodecs_module_state *st = get_module_state(self);
for (int i = 0; i < st->num_codecs; i++){
const MultibyteCodec *codec = &st->codec_list[i];
if (strcmp(codec->encoding, enc) == 0){
return _getcodec(codec);
}
}

return r;
PyErr_SetString(PyExc_LookupError,
"no such codec is supported.");
return NULL;
}

static int add_mappings(cjkcodecs_module_state *);
static int add_codecs(cjkcodecs_module_state *);

static int
register_maps(PyObject *module)
{
const struct dbcs_map *h;
// Init module state.
cjkcodecs_module_state *st = get_module_state(module);
if (add_mappings(st) < 0){
return -1;
}
if (add_codecs(st) < 0){
return -1;
}

for (h = mapping_list; h->charset[0] != '\0' h++){
for (int i = 0; i < st->num_mappings; i++){
const struct dbcs_map *h = &st->mapping_list[i];
char mhname[256] = "__map_"
strcpy(mhname + sizeof("__map_") - 1, h->charset);

Expand DownExpand Up@@ -394,6 +447,13 @@ _cjk_exec(PyObject *module)
return register_maps(module);
}

static void
_cjk_free(void *mod)
{
cjkcodecs_module_state *st = get_module_state((PyObject *)mod);
PyMem_Free(st->mapping_list);
PyMem_Free(st->codec_list);
}

static struct PyMethodDef _cjk_methods[] ={
{"getcodec", (PyCFunction)getcodec, METH_O, ""},
Expand All@@ -409,9 +469,10 @@ static PyModuleDef_Slot _cjk_slots[] ={
static struct PyModuleDef _cjk_module ={\
PyModuleDef_HEAD_INIT, \
.m_name = "_codecs_"#loc, \
.m_size = 0, \
.m_size = sizeof(cjkcodecs_module_state), \
.m_methods = _cjk_methods, \
.m_slots = _cjk_slots, \
.m_free = _cjk_free, \
}; \
\
PyMODINIT_FUNC \
Expand Down
Loading