From 56bf3e0c7bcffc98f3af1e4c21249ea619752e1c Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Mon, 31 Oct 2022 19:54:34 +0100 Subject: [PATCH 01/12] Feature: Introduce C enum parsing using the CEnum class one create Python Enums easily using a C style syntax. ``` class Test(CEnum): __enum__ = """ A = 1, B = 2, C """ ``` Signed-off-by: Sophie Tyalie --- cstruct/__init__.py | 1 + cstruct/abstract.py | 76 ++++++++++++++++++++++++++++++++++++++++++++- cstruct/c_parser.py | 55 ++++++++++++++++++++++++++++++-- cstruct/cenum.py | 4 +++ tests/test_cenum.py | 17 ++++++++++ 5 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 cstruct/cenum.py create mode 100644 tests/test_cenum.py diff --git a/cstruct/__init__.py b/cstruct/__init__.py index d20062e..b8dd41b 100644 --- a/cstruct/__init__.py +++ b/cstruct/__init__.py @@ -43,6 +43,7 @@ from .cstruct import CStruct from .c_parser import parse_def from .mem_cstruct import MemCStruct +from .cenum import CEnum __all__ = [ 'LITTLE_ENDIAN', diff --git a/cstruct/abstract.py b/cstruct/abstract.py index f5aeca9..0843bd3 100644 --- a/cstruct/abstract.py +++ b/cstruct/abstract.py @@ -28,8 +28,10 @@ from collections import OrderedDict from typing import Any, BinaryIO, List, Dict, Optional, Type, Tuple, Union import hashlib +from enum import IntEnum, EnumMeta, _EnumDict +from unicodedata import name from .base import STRUCTS -from .c_parser import parse_struct, parse_def, Tokens +from .c_parser import parse_struct, parse_def, parse_enum, Tokens from .field import calculate_padding, FieldType from .exceptions import CStructException @@ -257,3 +259,75 @@ def __setstate__(self, state: bytes) -> bool: state: bytes to be unpacked """ return self.unpack(state) + + +class CEnumMeta(EnumMeta): + __size__: int = 0 + + class WrapperDict(_EnumDict): + def __setitem__(self, key: str, value: Any) -> None: + if key == "__enum__": + env = parse_enum(value, self["__qualname__"]) + for k, v in env["__constants__"].items(): + super().__setitem__(k, v) + else: + return super().__setitem__(key, value) + + @classmethod + def __prepare__(metacls, cls, bases, **kwds): + namespace = EnumMeta.__prepare__(cls, bases, **kwds) + namespace.__class__ = metacls.WrapperDict + return namespace + + def __len__(cls) -> int: + "Enum size (in bytes)" + return cls.__size__ + + @property + def size(cls) -> int: + "Enum size (in bytes)" + return cls.__size__ + +class AbstractCEnum(IntEnum, metaclass=CEnumMeta): + """ + Abstract C enum to Python class + """ + + @classmethod + def parse( + cls, + __enum__: Union[str, Tokens, Dict[str, Any]], + __name__: Optional[str] = None, + __size__: Optional[int] = None, + **kargs: Dict[str, Any] + ) -> Type["AbstractCEnum"]: + """ + Return a new Python Enum class mapping a C enum definition + + Args: + __enum__: Definition of the enum in C syntax + __name__: Name of the new Enum. If empty, a name based on the __enum__ hash is generated + + Returns: + cls: A new class mapping the definition + """ + + cls_kargs: Dict[str, Any] = dict(kargs) + if __size__ is not None: + cls_kargs['__size__'] = __size__ + + cls_kargs['__enum__'] = __enum__ + if isinstance(__enum__, (str, Tokens)): + del cls_kargs["__enum__"] + cls_kargs.update(parse_def(__enum__, __cls__=cls, **cls_kargs)) + cls_kargs["__enum__"] = None + elif isinstance(__enum__, dict): + del cls_kargs["__enum__"] + cls_kargs.update(__enum__) + cls_kargs["__enum__"] = None + if __name__ is None: + __name__ = cls.__name__ + "_" + hashlib.sha1(str(__enum__).encode("utf-8")).hexdigest() + cls_kargs["__anonymous__"] = True + cls_kargs["__name__"] = __name__ + return IntEnum(__name__, cls_kargs["__constants__"]) + return type(__name__, (cls,), cls_kargs) \ No newline at end of file diff --git a/cstruct/c_parser.py b/cstruct/c_parser.py index 476067a..aea2350 100644 --- a/cstruct/c_parser.py +++ b/cstruct/c_parser.py @@ -31,7 +31,7 @@ from .exceptions import CStructException, ParserError if TYPE_CHECKING: - from .abstract import AbstractCStruct + from .abstract import AbstractCStruct, AbstractCEnum __all__ = ['parse_struct', 'parse_def', 'Tokens'] @@ -52,7 +52,7 @@ def __init__(self, text: str) -> None: else: lines.append(line) text = " ".join(lines) - text = text.replace(";", " ; ").replace("{", " { ").replace("}", " } ") + text = text.replace(";", " ; ").replace("{", " { ").replace("}", " } ").replace(",", " , ").replace("=", " = ") self.tokens = text.split() def pop(self) -> str: @@ -168,6 +168,57 @@ def parse_def( else: raise ParserError("{} definition expected".format(vtype)) +def parse_enum( + __enum__: Union[str, Tokens], + __cls__: Type['AbstractCEnum'], + **kargs: Any +) -> Optional[Dict[str, Any]]: + constants: Dict[str, int] = OrderedDict() + + if isinstance(__enum__, Tokens): + tokens = __enum__ + else: + tokens = Tokens(__enum__) + + while len(tokens): + name = tokens.pop() + + next_token = tokens.pop() + if next_token == ",": # enum-constant without explicit value + if len(constants) == 0: + value = 0 + else: + value = next(reversed(constants.values())) + 1 + elif next_token == "=": + exp_elems = [] + next_token = tokens.pop() + while not next_token.endswith(","): + exp_elems.append(next_token) + if len(tokens) > 0: + next_token = tokens.pop() + else: + break + + if len(exp_elems) == 0: + raise ParserError(f"enum is missing value expression") + + int_expr = " ".join(exp_elems) + try: + value = c_eval(int_expr) + except (ValueError, TypeError): + value = int(int_expr) + else: + raise ParserError(f"{__enum__} is not a valid enum expression") + + if name in constants: + raise ParserError(f"duplicate enum name {name}") + constants[name] = value + + result = { + '__constants__': constants + } + return result + def parse_struct( __struct__: Union[str, Tokens], diff --git a/cstruct/cenum.py b/cstruct/cenum.py new file mode 100644 index 0000000..0fc1654 --- /dev/null +++ b/cstruct/cenum.py @@ -0,0 +1,4 @@ +from .abstract import AbstractCEnum + +class CEnum(AbstractCEnum): + ... \ No newline at end of file diff --git a/tests/test_cenum.py b/tests/test_cenum.py new file mode 100644 index 0000000..f137f8f --- /dev/null +++ b/tests/test_cenum.py @@ -0,0 +1,17 @@ +from cstruct import CEnum + +class Dummy(CEnum): + __enum__ = """ + A, + B, + C = 2, + D = 5 + 7, + E = 2 + """ + +def test_dummy(): + assert Dummy.A == 0 + assert Dummy.B == 1 + assert Dummy.C == 2 + assert Dummy.D == 12 + assert Dummy.E == 2 \ No newline at end of file From f37c266e3d52da5cb228f23dc725e1a672bef019 Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Mon, 31 Oct 2022 20:13:44 +0100 Subject: [PATCH 02/12] Feature: Allow CEnum to be initialized using __def__ - renamed `parse_def` to `parse_struct_def` to allow for `parse_enum_def` function Signed-off-by: Sophie Tyalie --- cstruct/__init__.py | 4 ++-- cstruct/abstract.py | 16 ++++++++++------ cstruct/c_parser.py | 40 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/cstruct/__init__.py b/cstruct/__init__.py index b8dd41b..6dd3ec2 100644 --- a/cstruct/__init__.py +++ b/cstruct/__init__.py @@ -41,7 +41,7 @@ ) from .abstract import CStructMeta, AbstractCStruct from .cstruct import CStruct -from .c_parser import parse_def +from .c_parser import parse_struct_def from .mem_cstruct import MemCStruct from .cenum import CEnum @@ -151,7 +151,7 @@ def parse( """ if __cls__ is None: __cls__ = CStruct - cls_def = parse_def(__struct__, __cls__=__cls__, **kargs) + cls_def = parse_struct_def(__struct__, __cls__=__cls__, **kargs) if cls_def is None: return None else: diff --git a/cstruct/abstract.py b/cstruct/abstract.py index 0843bd3..b997f9c 100644 --- a/cstruct/abstract.py +++ b/cstruct/abstract.py @@ -31,7 +31,7 @@ from enum import IntEnum, EnumMeta, _EnumDict from unicodedata import name from .base import STRUCTS -from .c_parser import parse_struct, parse_def, parse_enum, Tokens +from .c_parser import parse_struct, parse_struct_def, parse_enum_def,parse_enum, Tokens from .field import calculate_padding, FieldType from .exceptions import CStructException @@ -50,7 +50,7 @@ def __new__(metacls: Type[type], name: str, bases: Tuple[str], namespace: Dict[s namespace.update(parse_struct(**namespace)) __struct__ = True if '__def__' in namespace: - namespace.update(parse_def(**namespace)) + namespace.update(parse_struct_def(**namespace)) __struct__ = True # Create the new class new_class: Type[Any] = super().__new__(metacls, name, bases, namespace) @@ -132,7 +132,7 @@ def parse( cls_kargs['__struct__'] = __struct__ if isinstance(__struct__, (str, Tokens)): del cls_kargs['__struct__'] - cls_kargs.update(parse_def(__struct__, __cls__=cls, **cls_kargs)) + cls_kargs.update(parse_struct_def(__struct__, __cls__=cls, **cls_kargs)) cls_kargs['__struct__'] = None elif isinstance(__struct__, dict): del cls_kargs['__struct__'] @@ -266,8 +266,13 @@ class CEnumMeta(EnumMeta): class WrapperDict(_EnumDict): def __setitem__(self, key: str, value: Any) -> None: + env = None if key == "__enum__": env = parse_enum(value, self["__qualname__"]) + elif key == "__def__": + env = parse_enum_def(value, self["__qualname__"]) + + if env is not None: for k, v in env["__constants__"].items(): super().__setitem__(k, v) else: @@ -319,7 +324,7 @@ def parse( cls_kargs['__enum__'] = __enum__ if isinstance(__enum__, (str, Tokens)): del cls_kargs["__enum__"] - cls_kargs.update(parse_def(__enum__, __cls__=cls, **cls_kargs)) + cls_kargs.update(parse_enum_def(__enum__, __cls__=cls, **cls_kargs)) cls_kargs["__enum__"] = None elif isinstance(__enum__, dict): del cls_kargs["__enum__"] @@ -329,5 +334,4 @@ def parse( __name__ = cls.__name__ + "_" + hashlib.sha1(str(__enum__).encode("utf-8")).hexdigest() cls_kargs["__anonymous__"] = True cls_kargs["__name__"] = __name__ - return IntEnum(__name__, cls_kargs["__constants__"]) - return type(__name__, (cls,), cls_kargs) \ No newline at end of file + return IntEnum(__name__, cls_kargs["__constants__"]) \ No newline at end of file diff --git a/cstruct/c_parser.py b/cstruct/c_parser.py index aea2350..3d0049f 100644 --- a/cstruct/c_parser.py +++ b/cstruct/c_parser.py @@ -33,7 +33,7 @@ if TYPE_CHECKING: from .abstract import AbstractCStruct, AbstractCEnum -__all__ = ['parse_struct', 'parse_def', 'Tokens'] +__all__ = ['parse_struct', 'parse_struct_def', 'Tokens'] class Tokens(object): @@ -142,7 +142,7 @@ def parse_type(tokens: Tokens, __cls__: Type['AbstractCStruct'], byte_order: Opt return FieldType(kind, c_type, ref, vlen, flexible_array, byte_order, offset) -def parse_def( +def parse_struct_def( __def__: Union[str, Tokens], __cls__: Type['AbstractCStruct'], __byte_order__: Optional[str] = None, @@ -168,6 +168,33 @@ def parse_def( else: raise ParserError("{} definition expected".format(vtype)) + +def parse_enum_def( + __def__: Union[str, Tokens], + __cls__: Type["AbstractCEnum"], + **kargs: Any +) -> Optional[Dict[str, Any]]: + # naive C enum parsing + if isinstance(__def__, Tokens): + tokens = __def__ + else: + tokens = Tokens(__def__) + if not tokens: + return None + kind = tokens.pop() + if kind not in ['enum']: + raise ParserError(f"enum expected - {kind}") + + vtype = tokens.pop() + if tokens.get() == '{': # named enum + tokens.pop() + return parse_enum(tokens, __cls__=__cls__) + elif vtype == '{': + return parse_enum(tokens, __cls__=__cls__) + else: + raise ParserError(f"{vtype} definition expected") + + def parse_enum( __enum__: Union[str, Tokens], __cls__: Type['AbstractCEnum'], @@ -181,6 +208,10 @@ def parse_enum( tokens = Tokens(__enum__) while len(tokens): + if tokens.get() == '}': + tokens.pop() + break + name = tokens.pop() next_token = tokens.pop() @@ -192,7 +223,7 @@ def parse_enum( elif next_token == "=": exp_elems = [] next_token = tokens.pop() - while not next_token.endswith(","): + while next_token not in {",", "}"}: exp_elems.append(next_token) if len(tokens) > 0: next_token = tokens.pop() @@ -207,6 +238,9 @@ def parse_enum( value = c_eval(int_expr) except (ValueError, TypeError): value = int(int_expr) + + if next_token == "}": + break else: raise ParserError(f"{__enum__} is not a valid enum expression") From 3fdfd64259f94ee8ff91049afbfac4d91c0a198c Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Tue, 1 Nov 2022 11:18:31 +0100 Subject: [PATCH 03/12] enum: Register enums in base.ENUMS Using Pythons metaclass new function, we look for valid C Enum classes (__enum__ = True) and register them there in the `base.ENUMS` variable - `__size__` is a required parameter for Enums - introduced `CEnumException` Signed-off-by: Sophie Tyalie --- cstruct/abstract.py | 21 +++++++++++++++++---- cstruct/base.py | 4 +++- cstruct/exceptions.py | 3 +++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/cstruct/abstract.py b/cstruct/abstract.py index b997f9c..bf88060 100644 --- a/cstruct/abstract.py +++ b/cstruct/abstract.py @@ -28,12 +28,13 @@ from collections import OrderedDict from typing import Any, BinaryIO, List, Dict, Optional, Type, Tuple, Union import hashlib +import sys from enum import IntEnum, EnumMeta, _EnumDict from unicodedata import name -from .base import STRUCTS +from .base import STRUCTS, ENUMS from .c_parser import parse_struct, parse_struct_def, parse_enum_def,parse_enum, Tokens from .field import calculate_padding, FieldType -from .exceptions import CStructException +from .exceptions import CStructException, CEnumException __all__ = ['CStructMeta', 'AbstractCStruct'] @@ -262,8 +263,6 @@ def __setstate__(self, state: bytes) -> bool: class CEnumMeta(EnumMeta): - __size__: int = 0 - class WrapperDict(_EnumDict): def __setitem__(self, key: str, value: Any) -> None: env = None @@ -273,8 +272,13 @@ def __setitem__(self, key: str, value: Any) -> None: env = parse_enum_def(value, self["__qualname__"]) if env is not None: + # register the enum constants in the object namespace, + # using the Python Enum class Namespace dict that does the + # heavy lifting for k, v in env["__constants__"].items(): super().__setitem__(k, v) + + super().__setitem__("__enum__", True) else: return super().__setitem__(key, value) @@ -284,6 +288,15 @@ def __prepare__(metacls, cls, bases, **kwds): namespace.__class__ = metacls.WrapperDict return namespace + def __new__(metacls: type["CEnumMeta"], cls: str, bases: tuple[type, ...], classdict: _EnumDict, **kwds: Any) -> "CEnumMeta": + inst = super().__new__(metacls, cls, bases, classdict, **kwds) + + if classdict.get("__enum__", False): + if "__size__" not in classdict: + raise CEnumException("__size__ not specified. Cannot derive size as it is architecture dependent") + ENUMS[cls] = inst + return inst + def __len__(cls) -> int: "Enum size (in bytes)" return cls.__size__ diff --git a/cstruct/base.py b/cstruct/base.py index b350a67..f9ca0f1 100644 --- a/cstruct/base.py +++ b/cstruct/base.py @@ -25,7 +25,7 @@ from typing import Any, Dict, Type, TYPE_CHECKING if TYPE_CHECKING: - from .abstract import AbstractCStruct + from .abstract import AbstractCStruct, AbstractCEnum __all__ = [ 'LITTLE_ENDIAN', @@ -48,6 +48,8 @@ STRUCTS: Dict[str, Type["AbstractCStruct"]] = {} +ENUMS: Dict[str, Type["AbstractCEnum"]] = {} + DEFINES: Dict[str, Any] = {} TYPEDEFS: Dict[str, str] = { diff --git a/cstruct/exceptions.py b/cstruct/exceptions.py index 65cd668..3d66fb3 100644 --- a/cstruct/exceptions.py +++ b/cstruct/exceptions.py @@ -23,11 +23,14 @@ # __all__ = [ + "CEnumException", "CStructException", "ParserError", "EvalError", ] +class CEnumException(Exception): + pass class CStructException(Exception): pass From 601395e58fa21bf189192c725ca7d5087311b23e Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Tue, 1 Nov 2022 11:24:15 +0100 Subject: [PATCH 04/12] Removed __cls__ parameter in enum parsing It wasn't used so far and I'm not sure if that will change. Enum parsing is far more simple than struct parsing in this regard Signed-off-by: Sophie Tyalie --- cstruct/abstract.py | 4 ++-- cstruct/c_parser.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cstruct/abstract.py b/cstruct/abstract.py index bf88060..8c17523 100644 --- a/cstruct/abstract.py +++ b/cstruct/abstract.py @@ -267,9 +267,9 @@ class WrapperDict(_EnumDict): def __setitem__(self, key: str, value: Any) -> None: env = None if key == "__enum__": - env = parse_enum(value, self["__qualname__"]) + env = parse_enum(value) elif key == "__def__": - env = parse_enum_def(value, self["__qualname__"]) + env = parse_enum_def(value) if env is not None: # register the enum constants in the object namespace, diff --git a/cstruct/c_parser.py b/cstruct/c_parser.py index 3d0049f..0039057 100644 --- a/cstruct/c_parser.py +++ b/cstruct/c_parser.py @@ -171,7 +171,6 @@ def parse_struct_def( def parse_enum_def( __def__: Union[str, Tokens], - __cls__: Type["AbstractCEnum"], **kargs: Any ) -> Optional[Dict[str, Any]]: # naive C enum parsing @@ -188,16 +187,15 @@ def parse_enum_def( vtype = tokens.pop() if tokens.get() == '{': # named enum tokens.pop() - return parse_enum(tokens, __cls__=__cls__) + return parse_enum(tokens) elif vtype == '{': - return parse_enum(tokens, __cls__=__cls__) + return parse_enum(tokens) else: raise ParserError(f"{vtype} definition expected") def parse_enum( __enum__: Union[str, Tokens], - __cls__: Type['AbstractCEnum'], **kargs: Any ) -> Optional[Dict[str, Any]]: constants: Dict[str, int] = OrderedDict() From 58d2c8f142d8568034f5c100fb224349a7f8236f Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Tue, 1 Nov 2022 11:46:17 +0100 Subject: [PATCH 05/12] Enum: Allow parsing of enums in struct definitions Previously defined named CEnums are correctly parsed when used in a struct definition now. It is unclear whether unnamed and `AbstractCEnum.parse` enums can also be used. Presumably not. Signed-off-by: Sophie Tyalie --- cstruct/base.py | 7 +++++++ cstruct/c_parser.py | 13 ++++++++++--- cstruct/field.py | 24 ++++++++++++++++++++---- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/cstruct/base.py b/cstruct/base.py index f9ca0f1..ab21666 100644 --- a/cstruct/base.py +++ b/cstruct/base.py @@ -93,4 +93,11 @@ 'uint64': 'Q', } +ENUM_SIZE_TO_C_TYPE: Dict[int, str] = { + 1: 'uint8', + 2: 'uint16', + 4: 'uint32', + 8: 'uint64' +} + CHAR_ZERO = bytes('\0', 'ascii') diff --git a/cstruct/c_parser.py b/cstruct/c_parser.py index 0039057..f43dc60 100644 --- a/cstruct/c_parser.py +++ b/cstruct/c_parser.py @@ -25,7 +25,7 @@ import re from collections import OrderedDict from typing import Union, Optional, Any, Dict, Type, TYPE_CHECKING -from .base import DEFINES, TYPEDEFS, STRUCTS +from .base import DEFINES, ENUMS, TYPEDEFS, STRUCTS from .field import calculate_padding, Kind, FieldType from .c_expr import c_eval from .exceptions import CStructException, ParserError @@ -76,9 +76,9 @@ def parse_type(tokens: Tokens, __cls__: Type['AbstractCStruct'], byte_order: Opt raise ParserError("Parsing error") c_type = tokens.pop() # signed/unsigned/struct - if c_type in ['signed', 'unsigned', 'struct', 'union'] and len(tokens) > 1: + if c_type in ['signed', 'unsigned', 'struct', 'union', 'enum'] and len(tokens) > 1: c_type = c_type + " " + tokens.pop() - + vlen = 1 flexible_array = False @@ -136,6 +136,13 @@ def parse_type(tokens: Tokens, __cls__: Type['AbstractCStruct'], byte_order: Opt ref = STRUCTS[tail] except KeyError: raise ParserError("Unknow {} {}".format(c_type, tail)) + elif c_type.startswith('enum'): + c_type, tail = c_type.split(' ', 1) + kind = Kind.ENUM + try: + ref = ENUMS[tail] + except KeyError: + raise ParserError(f"Unknown '{c_type}' '{tail}'") else: # other types kind = Kind.NATIVE ref = None diff --git a/cstruct/field.py b/cstruct/field.py index 2597b83..265fb0f 100644 --- a/cstruct/field.py +++ b/cstruct/field.py @@ -26,7 +26,7 @@ import struct from enum import Enum from typing import Optional, Any, List, Type, TYPE_CHECKING -from .base import NATIVE_ORDER, C_TYPE_TO_FORMAT +from .base import NATIVE_ORDER, C_TYPE_TO_FORMAT, ENUM_SIZE_TO_C_TYPE from .exceptions import ParserError if TYPE_CHECKING: @@ -58,6 +58,8 @@ class Kind(Enum): "Struct type" UNION = 2 "Union type" + ENUM = 3 + "Enum type" class FieldType(object): @@ -115,8 +117,12 @@ def unpack_from(self, buffer: bytes, offset: int = 0) -> Any: Returns: data: The unpacked data """ - if self.is_native: + if self.is_native or self.is_enum: result = struct.unpack_from(self.fmt, buffer, self.offset + offset) + + if self.is_enum: + result = tuple(map(self.ref, result)) + if self.is_array: return list(result) else: @@ -162,6 +168,11 @@ def is_native(self) -> bool: "True if the field is a native type (e.g. int, char)" return self.kind == Kind.NATIVE + @property + def is_enum(self) -> bool: + "True if the field is an enum" + return self.kind == Kind.ENUM + @property def is_struct(self) -> bool: "True if the field is a struct" @@ -180,13 +191,18 @@ def native_format(self) -> str: return C_TYPE_TO_FORMAT[self.c_type] except KeyError: raise ParserError("Unknow type {}".format(self.c_type)) + elif self.is_enum: + try: + return C_TYPE_TO_FORMAT[ENUM_SIZE_TO_C_TYPE[len(self.ref)]] + except KeyError: + raise ParserError(f"Enum has invalid size. Needs to be in {ENUM_SIZE_TO_C_TYPE.keys()}") else: return 'c' @property def fmt(self) -> str: "Field format prefixed by byte order (struct library format)" - if self.is_native: + if self.is_native or self.is_enum: fmt = (str(self.vlen) if self.vlen > 1 or self.flexible_array else '') + self.native_format else: # Struct/Union fmt = str(self.vlen * self.ref.sizeof()) + self.native_format @@ -203,7 +219,7 @@ def vsize(self) -> int: @property def alignment(self) -> int: "Alignment" - if self.is_native: + if self.is_native or self.is_enum: if self.byte_order is not None: return struct.calcsize(self.byte_order + self.native_format) else: From aed0d7269ee59df6934f7499f49b4b5f42f32f2a Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Tue, 1 Nov 2022 11:57:48 +0100 Subject: [PATCH 06/12] Enum: Fix parser Parser couldn't handle the case that an enum-constant without explicit value and no following `,` was the last thing in the enum definition. Example error case ``` enum { A } ``` Signed-off-by: Sophie Tyalie --- cstruct/c_parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cstruct/c_parser.py b/cstruct/c_parser.py index f43dc60..1727829 100644 --- a/cstruct/c_parser.py +++ b/cstruct/c_parser.py @@ -220,7 +220,8 @@ def parse_enum( name = tokens.pop() next_token = tokens.pop() - if next_token == ",": # enum-constant without explicit value + print(name, next_token) + if next_token in {",", "}"}: # enum-constant without explicit value if len(constants) == 0: value = 0 else: From 26e36a65fc0ed9b5533faf6973ee763432bd0c82 Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Tue, 1 Nov 2022 12:23:14 +0100 Subject: [PATCH 07/12] Enum: Fix `AbstractCEnum.parse` This function was not in a working state previously. While making the function work a few things needed to be changed too. For example should `len(enum)` return the number of elements instead of the size for CEnums and such. Introduced default enum size of 4 bytes Signed-off-by: Sophie Tyalie --- cstruct/abstract.py | 29 ++++++++++------------------- cstruct/base.py | 1 + cstruct/field.py | 2 +- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/cstruct/abstract.py b/cstruct/abstract.py index 8c17523..616acb5 100644 --- a/cstruct/abstract.py +++ b/cstruct/abstract.py @@ -31,12 +31,12 @@ import sys from enum import IntEnum, EnumMeta, _EnumDict from unicodedata import name -from .base import STRUCTS, ENUMS +from .base import STRUCTS, ENUMS, DEFAULT_ENUM_SIZE from .c_parser import parse_struct, parse_struct_def, parse_enum_def,parse_enum, Tokens from .field import calculate_padding, FieldType from .exceptions import CStructException, CEnumException -__all__ = ['CStructMeta', 'AbstractCStruct'] +__all__ = ['CStructMeta', 'AbstractCStruct', "CEnumMeta", 'AbstractCEnum'] class CStructMeta(ABCMeta): @@ -277,8 +277,6 @@ def __setitem__(self, key: str, value: Any) -> None: # heavy lifting for k, v in env["__constants__"].items(): super().__setitem__(k, v) - - super().__setitem__("__enum__", True) else: return super().__setitem__(key, value) @@ -291,16 +289,13 @@ def __prepare__(metacls, cls, bases, **kwds): def __new__(metacls: type["CEnumMeta"], cls: str, bases: tuple[type, ...], classdict: _EnumDict, **kwds: Any) -> "CEnumMeta": inst = super().__new__(metacls, cls, bases, classdict, **kwds) - if classdict.get("__enum__", False): + if len(inst) > 0: if "__size__" not in classdict: raise CEnumException("__size__ not specified. Cannot derive size as it is architecture dependent") - ENUMS[cls] = inst + if not classdict.get("__anonymous__", False): + ENUMS[cls] = inst return inst - def __len__(cls) -> int: - "Enum size (in bytes)" - return cls.__size__ - @property def size(cls) -> int: "Enum size (in bytes)" @@ -331,20 +326,16 @@ def parse( """ cls_kargs: Dict[str, Any] = dict(kargs) - if __size__ is not None: - cls_kargs['__size__'] = __size__ + cls_kargs['__size__'] = DEFAULT_ENUM_SIZE if __size__ is None else __size__ - cls_kargs['__enum__'] = __enum__ if isinstance(__enum__, (str, Tokens)): - del cls_kargs["__enum__"] cls_kargs.update(parse_enum_def(__enum__, __cls__=cls, **cls_kargs)) - cls_kargs["__enum__"] = None elif isinstance(__enum__, dict): - del cls_kargs["__enum__"] cls_kargs.update(__enum__) - cls_kargs["__enum__"] = None + if __name__ is None: __name__ = cls.__name__ + "_" + hashlib.sha1(str(__enum__).encode("utf-8")).hexdigest() cls_kargs["__anonymous__"] = True - cls_kargs["__name__"] = __name__ - return IntEnum(__name__, cls_kargs["__constants__"]) \ No newline at end of file + + cls_kargs.update(cls_kargs["__constants__"]) + return cls(__name__, cls_kargs) \ No newline at end of file diff --git a/cstruct/base.py b/cstruct/base.py index ab21666..74e569f 100644 --- a/cstruct/base.py +++ b/cstruct/base.py @@ -101,3 +101,4 @@ } CHAR_ZERO = bytes('\0', 'ascii') +DEFAULT_ENUM_SIZE = 4 \ No newline at end of file diff --git a/cstruct/field.py b/cstruct/field.py index 265fb0f..db4f6df 100644 --- a/cstruct/field.py +++ b/cstruct/field.py @@ -193,7 +193,7 @@ def native_format(self) -> str: raise ParserError("Unknow type {}".format(self.c_type)) elif self.is_enum: try: - return C_TYPE_TO_FORMAT[ENUM_SIZE_TO_C_TYPE[len(self.ref)]] + return C_TYPE_TO_FORMAT[ENUM_SIZE_TO_C_TYPE[self.ref.size]] except KeyError: raise ParserError(f"Enum has invalid size. Needs to be in {ENUM_SIZE_TO_C_TYPE.keys()}") else: From 821a99a20fae0d366ac4bc762a6c47b8824f0011 Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Tue, 1 Nov 2022 12:26:56 +0100 Subject: [PATCH 08/12] Enum: Remove hard size enforcement A warning will now be displayed and the enum size will be set to `base.DEFAULT_ENUM_SIZE` instead Signed-off-by: Sophie Tyalie --- cstruct/abstract.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cstruct/abstract.py b/cstruct/abstract.py index 616acb5..f8ccaff 100644 --- a/cstruct/abstract.py +++ b/cstruct/abstract.py @@ -291,7 +291,8 @@ def __new__(metacls: type["CEnumMeta"], cls: str, bases: tuple[type, ...], class if len(inst) > 0: if "__size__" not in classdict: - raise CEnumException("__size__ not specified. Cannot derive size as it is architecture dependent") + inst.__size__ = DEFAULT_ENUM_SIZE + print(f"Warning: __size__ not specified for enum {cls}. Will default to {DEFAULT_ENUM_SIZE} bytes") if not classdict.get("__anonymous__", False): ENUMS[cls] = inst return inst @@ -320,13 +321,15 @@ def parse( Args: __enum__: Definition of the enum in C syntax __name__: Name of the new Enum. If empty, a name based on the __enum__ hash is generated + __size__: Number of bytes that the enum should be read as Returns: cls: A new class mapping the definition """ cls_kargs: Dict[str, Any] = dict(kargs) - cls_kargs['__size__'] = DEFAULT_ENUM_SIZE if __size__ is None else __size__ + if __size__ is not None: + cls_kargs['__size__'] = __size__ if isinstance(__enum__, (str, Tokens)): cls_kargs.update(parse_enum_def(__enum__, __cls__=cls, **cls_kargs)) From 842c77b3b058acfde8efa15bb48ac007524e3bc1 Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Tue, 1 Nov 2022 12:42:35 +0100 Subject: [PATCH 09/12] Enum: Allow inline definition of enums This includes named and unnamed enums that are defined in a struct, similar to this: ``` struct { enum {A, B, C} v; } ``` Signed-off-by: Sophie Tyalie --- cstruct/c_parser.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/cstruct/c_parser.py b/cstruct/c_parser.py index 1727829..f82a6a5 100644 --- a/cstruct/c_parser.py +++ b/cstruct/c_parser.py @@ -137,12 +137,24 @@ def parse_type(tokens: Tokens, __cls__: Type['AbstractCStruct'], byte_order: Opt except KeyError: raise ParserError("Unknow {} {}".format(c_type, tail)) elif c_type.startswith('enum'): + #from .abstract import AbstractCEnum + from .cenum import CEnum + c_type, tail = c_type.split(' ', 1) kind = Kind.ENUM - try: - ref = ENUMS[tail] - except KeyError: - raise ParserError(f"Unknown '{c_type}' '{tail}'") + if tokens.get() == '{': # Named nested struct + tokens.push(tail) + tokens.push(c_type) + ref = CEnum.parse(tokens, __name__=tail) + elif tail == '{': # unnamed nested struct + tokens.push(tail) + tokens.push(c_type) + ref = CEnum.parse(tokens) + else: + try: + ref = ENUMS[tail] + except KeyError: + raise ParserError(f"Unknown '{c_type}' '{tail}'") else: # other types kind = Kind.NATIVE ref = None @@ -254,6 +266,9 @@ def parse_enum( raise ParserError(f"duplicate enum name {name}") constants[name] = value + if next_token == "}": + break + result = { '__constants__': constants } From b1db0568ed6d7655ae37cfd25b23aaca9f46b96a Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Tue, 1 Nov 2022 12:47:01 +0100 Subject: [PATCH 10/12] Enum: Fix enum parser I'm not fully sure what the test case was that would have resulted in an error, but looking over the code there was a break at the wrong position that could have resulted in an unparsed enum constant. Signed-off-by: Sophie Tyalie --- cstruct/c_parser.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cstruct/c_parser.py b/cstruct/c_parser.py index f82a6a5..3cad8f6 100644 --- a/cstruct/c_parser.py +++ b/cstruct/c_parser.py @@ -232,13 +232,12 @@ def parse_enum( name = tokens.pop() next_token = tokens.pop() - print(name, next_token) if next_token in {",", "}"}: # enum-constant without explicit value if len(constants) == 0: value = 0 else: value = next(reversed(constants.values())) + 1 - elif next_token == "=": + elif next_token == "=": # enum-constant with explicit value exp_elems = [] next_token = tokens.pop() while next_token not in {",", "}"}: @@ -256,9 +255,6 @@ def parse_enum( value = c_eval(int_expr) except (ValueError, TypeError): value = int(int_expr) - - if next_token == "}": - break else: raise ParserError(f"{__enum__} is not a valid enum expression") From 230f80a0a8fb261a80c8659e5a166883e4ed9ec6 Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Tue, 1 Nov 2022 13:09:21 +0100 Subject: [PATCH 11/12] Enum: use signed instead of unsigned int for enum representation This hits upon a larger issue. It doesn't seem fully clear what type a compiler will use to represent an enum, but signed integer seems to be a good enough solution as enum-constants (not the enum itself) are defined as int signed. See following patch note https://gcc.gnu.org/legacy-ml/gcc-patches/2000-07/msg00993.html Signed-off-by: Sophie Tyalie --- cstruct/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cstruct/base.py b/cstruct/base.py index 74e569f..6b427ae 100644 --- a/cstruct/base.py +++ b/cstruct/base.py @@ -94,10 +94,10 @@ } ENUM_SIZE_TO_C_TYPE: Dict[int, str] = { - 1: 'uint8', - 2: 'uint16', - 4: 'uint32', - 8: 'uint64' + 1: 'int8', + 2: 'int16', + 4: 'int32', + 8: 'int64' } CHAR_ZERO = bytes('\0', 'ascii') From 6ed7884f08bde271fb4eb9b6ee14468332f8b2ef Mon Sep 17 00:00:00 2001 From: Sophie Tyalie Date: Tue, 1 Nov 2022 13:14:22 +0100 Subject: [PATCH 12/12] Enum: Fix pytest errors with older python versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python 3.10 supports using `type[…]` or `tuple[…]` as type hints. This is relatively new and as such not support in older pythons Signed-off-by: Sophie Tyalie --- cstruct/abstract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cstruct/abstract.py b/cstruct/abstract.py index f8ccaff..88e8493 100644 --- a/cstruct/abstract.py +++ b/cstruct/abstract.py @@ -286,7 +286,7 @@ def __prepare__(metacls, cls, bases, **kwds): namespace.__class__ = metacls.WrapperDict return namespace - def __new__(metacls: type["CEnumMeta"], cls: str, bases: tuple[type, ...], classdict: _EnumDict, **kwds: Any) -> "CEnumMeta": + def __new__(metacls: Type["CEnumMeta"], cls: str, bases: Tuple[Type, ...], classdict: _EnumDict, **kwds: Any) -> "CEnumMeta": inst = super().__new__(metacls, cls, bases, classdict, **kwds) if len(inst) > 0: