From ef6329af7c7be77b537fbfc5a5cc7105acc679f7 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 31 Jan 2021 12:46:00 -0800 Subject: [PATCH] Create a system to map between info.json and config.h/rules.mk (#11548) * generate rules.mk from a json mapping * generate rules.mk from a json mapping * support for config.h from json maps * improve the mapping system * document the mapping system * move data/maps to data/mappings * fix flake8 errors * fixup LED_MATRIX_DRIVER * remove product and description from the vision_division keymap level * reduce the complexity of generate-rules-mk * add tests for the generate commands * fix qmk doctor when submodules are not clean --- bin/qmk | 3 +- data/mappings/info_config.json | 42 +++ data/mappings/info_rules.json | 15 + docs/data_driven_config.md | 44 ++- keyboards/handwired/pytest/basic/info.json | 10 + .../vision_division/keymaps/default/config.h | 1 - lib/python/qmk/cli/doctor.py | 4 +- lib/python/qmk/cli/generate/config_h.py | 239 +++--------- lib/python/qmk/cli/generate/rules_mk.py | 61 +-- lib/python/qmk/constants.py | 2 +- lib/python/qmk/info.py | 347 ++++++------------ lib/python/qmk/tests/test_cli_commands.py | 29 ++ requirements.txt | 1 + 13 files changed, 338 insertions(+), 460 deletions(-) create mode 100644 data/mappings/info_config.json create mode 100644 data/mappings/info_rules.json create mode 100644 keyboards/handwired/pytest/basic/info.json diff --git a/bin/qmk b/bin/qmk index 801852d4e8..a3c1be328a 100755 --- a/bin/qmk +++ b/bin/qmk @@ -27,7 +27,8 @@ def _check_modules(requirements): line = line.split('#')[0] module = dict() - module['name'] = module['import'] = line.split('=')[0] if '=' in line else line + module['name'] = line.split('=')[0] if '=' in line else line + module['import'] = module['name'].replace('-', '_') # Not every module is importable by its own name. if module['name'] == "pep8-naming": diff --git a/data/mappings/info_config.json b/data/mappings/info_config.json new file mode 100644 index 0000000000..885e6d0256 --- /dev/null +++ b/data/mappings/info_config.json @@ -0,0 +1,42 @@ +# This file maps keys between `config.h` and `info.json`. It is used by QMK +# to correctly and consistently map back and forth between the two systems. +{ + # Format: + # : {"info_key": , ["value_type": ], ["to_json": ], ["to_c": ]} + # value_type: one of "array", "array.int", "int", "hex", "list", "mapping" + # to_json: Default `true`. Set to `false` to exclude this mapping from info.json + # to_c: Default `true`. Set to `false` to exclude this mapping from config.h + # warn_duplicate: Default `true`. Set to `false` to turn off warning when a value exists in both places + "DEBOUNCE": {"info_key": "debounce", "value_type": "int"} + "DEVICE_VER": {"info_key": "usb.device_ver", "value_type": "hex"}, + "DESCRIPTION": {"info_key": "keyboard_folder", "to_json": false}, + "DIODE_DIRECTION": {"info_key": "diode_direction"}, + "LAYOUTS": {"info_key": "layout_aliases", "value_type": "mapping"}, + "LED_CAPS_LOCK_PIN": {"info_key": "indicators.caps_lock"}, + "LED_NUM_LOCK_PIN": {"info_key": "indicators.num_lock"}, + "LED_SCROLL_LOCK_PIN": {"info_key": "indicators.scroll_lock"}, + "MANUFACTURER": {"info_key": "manufacturer"}, + "RGB_DI_PIN": {"info_key": "rgblight.pin"}, + "RGBLED_NUM": {"info_key": "rgblight.led_count", "value_type": "int"}, + "RGBLED_SPLIT": {"info_key": "rgblight.split_count", "value_type": "array.int"}, + "RGBLIGHT_ANIMATIONS": {"info_key": "rgblight.animations.all", "value_type": "bool"}, + "RGBLIGHT_EFFECT_ALTERNATING": {"info_key": "rgblight.animations.alternating", "value_type": "bool"}, + "RGBLIGHT_EFFECT_BREATHING": {"info_key": "rgblight.animations.breathing", "value_type": "bool"}, + "RGBLIGHT_EFFECT_CHRISTMAS": {"info_key": "rgblight.animations.christmas", "value_type": "bool"}, + "RGBLIGHT_EFFECT_KNIGHT": {"info_key": "rgblight.animations.knight", "value_type": "bool"}, + "RGBLIGHT_EFFECT_RAINBOW_MOOD": {"info_key": "rgblight.animations.rainbow_mood", "value_type": "bool"}, + "RGBLIGHT_EFFECT_RAINBOW_SWIRL": {"info_key": "rgblight.animations.rainbow_swirl", "value_type": "bool"}, + "RGBLIGHT_EFFECT_RGB_TEST": {"info_key": "rgblight.animations.rgb_test", "value_type": "bool"}, + "RGBLIGHT_EFFECT_SNAKE": {"info_key": "rgblight.animations.snake", "value_type": "bool"}, + "RGBLIGHT_EFFECT_STATIC_GRADIENT": {"info_key": "rgblight.animations.static_gradient", "value_type": "bool"}, + "RGBLIGHT_EFFECT_TWINKLE": {"info_key": "rgblight.animations.twinkle"}, + "RGBLIGHT_LIMIT_VAL": {"info_key": "rgblight.max_brightness", "value_type": "int"}, + "RGBLIGHT_HUE_STEP": {"info_key": "rgblight.hue_steps", "value_type": "int"}, + "RGBLIGHT_SAT_STEP": {"info_key": "rgblight.saturation_steps", "value_type": "int"}, + "RGBLIGHT_VAL_STEP": {"info_key": "rgblight.brightness_steps", "value_type": "int"}, + "RGBLIGHT_SLEEP": {"info_key": "rgblight.sleep", "value_type": "bool"}, + "RGBLIGHT_SPLIT": {"info_key": "rgblight.split", "value_type": "bool"}, + "PRODUCT": {"info_key": "keyboard_folder", "to_json": false}, + "PRODUCT_ID": {"info_key": "usb.pid", "value_type": "hex"}, + "VENDOR_ID": {"info_key": "usb.vid", "value_type": "hex"} +} diff --git a/data/mappings/info_rules.json b/data/mappings/info_rules.json new file mode 100644 index 0000000000..97f772c4d5 --- /dev/null +++ b/data/mappings/info_rules.json @@ -0,0 +1,15 @@ +# This file maps keys between `rules.mk` and `info.json`. It is used by QMK +# to correctly and consistently map back and forth between the two systems. +{ + # Format: + # : {"info_key": , ["value_type": ], ["to_json": ], ["to_c": ]} + # value_type: one of "array", "array.int", "int", "list", "hex", "mapping" + # to_json: Default `true`. Set to `false` to exclude this mapping from info.json + # to_c: Default `true`. Set to `false` to exclude this mapping from rules.mk + # warn_duplicate: Default `true`. Set to `false` to turn off warning when a value exists in both places + "BOARD": {"info_key": "board"}, + "BOOTLOADER": {"info_key": "bootloader", "warn_duplicate": false}, + "LAYOUTS": {"info_key": "community_layouts", "value_type": "list"}, + "LED_MATRIX_DRIVER": {"info_key": "led_matrix.driver"}, + "MCU": {"info_key": "processor", "warn_duplicate": false}, +} diff --git a/docs/data_driven_config.md b/docs/data_driven_config.md index 7e4f232846..c2ad4fed8f 100644 --- a/docs/data_driven_config.md +++ b/docs/data_driven_config.md @@ -12,17 +12,18 @@ Now we have support for generating `rules.mk` and `config.h` values from `info.j ## Overview -On the C side of things nothing really changes. When you need to create a new rule or define you follow the same process: +On the C side of things nothing changes. When you need to create a new rule or define you follow the same process: 1. Add it to `docs/config_options.md` 1. Set a default in the appropriate core file -1. Add your `ifdef` and/or `#ifdef` statements as needed +1. Add your ifdef statements as needed You will then need to add support for your new configuration to `info.json`. The basic process is: 1. Add it to the schema in `data/schemas/keyboards.jsonschema` -1. Add code to extract it from `config.h`/`rules.mk` to `lib/python/qmk/info.py` -1. Add code to generate it to one of: +1. Add a mapping in `data/maps` +1. (optional and discoraged) Add code to extract/generate it to: + * `lib/python/qmk/info.py` * `lib/python/qmk/cli/generate/config_h.py` * `lib/python/qmk/cli/generate/rules_mk.py` @@ -32,12 +33,43 @@ This section describes adding support for a `config.h`/`rules.mk` value to info. ### Add it to the schema -QMK maintains schema files in `data/schemas`. The values that go into keyboard-specific `info.json` files are kept in `keyboard.jsonschema`. Any value you want to make available to end users to edit must go in here. +QMK maintains [jsonschema](https://json-schema.org/) files in `data/schemas`. The values that go into keyboard-specific `info.json` files are kept in `keyboard.jsonschema`. Any value you want to make available to end users to edit must go in here. -In some cases you can simply add a new top-level key. Some examples to follow are `keyboard_name`, `maintainer`, `processor`, and `url`. This is appropriate when your option is self-contained and not directly related to other options. In other cases you should group like options together in an `object`. This is particularly true when adding support for a feature. Some examples to follow for this are `indicators`, `matrix_pins`, and `rgblight`. If you are not sure how to integrate your new option(s) [open an issue](https://github.com/qmk/qmk_firmware/issues/new?assignees=&labels=cli%2C+python&template=other_issues.md&title=) or [join #cli on Discord](https://discord.gg/heQPAgy) and start a conversation there. +In some cases you can simply add a new top-level key. Some examples to follow are `keyboard_name`, `maintainer`, `processor`, and `url`. This is appropriate when your option is self-contained and not directly related to other options. + +In other cases you should group like options together in an `object`. This is particularly true when adding support for a feature. Some examples to follow for this are `indicators`, `matrix_pins`, and `rgblight`. If you are not sure how to integrate your new option(s) [open an issue](https://github.com/qmk/qmk_firmware/issues/new?assignees=&labels=cli%2C+python&template=other_issues.md&title=) or [join #cli on Discord](https://discord.gg/heQPAgy) and start a conversation there. + +### Add a mapping + +In most cases you can add a simple mapping. These are maintained as JSON files in `data/mappings/info_config.json` and `data/mappings/info_rules.json`, and control mapping for `config.h` and `rules.mk`, respectively. Each mapping is keyed by the `config.h` or `rules.mk` variable, and the value is a hash with the following keys: + +* `info_key`: (required) The location within `info.json` for this value. See below. +* `value_type`: (optional) Default `str`. The format for this variable's value. See below. +* `to_json`: (optional) Default `true`. Set to `false` to exclude this mapping from info.json +* `to_c`: (optional) Default `true`. Set to `false` to exclude this mapping from config.h +* `warn_duplicate`: (optional) Default `true`. Set to `false` to turn off warning when a value exists in both places + +#### Info Key + +We use JSON dot notation to address variables within info.json. For example, to access `info_json["rgblight"]["split_count"]` I would specify `rgblight.split_count`. This allows you to address deeply nested keys with a simple string. + +Under the hood we use [Dotty Dict](https://dotty-dict.readthedocs.io/en/latest/), you can refer to that documentation for how these strings are converted to object access. + +#### Value Types + +By default we treat all values as simple strings. If your value is more complex you can use one of these types to intelligently parse the data: + +* `array`: A comma separated array of strings +* `array.int`: A comma separated array of integers +* `int`: An integer +* `hex`: A number formatted as hex +* `list`: A space separate array of strings +* `mapping`: A hash of key/value pairs ### Add code to extract it +Most use cases can be solved by the mapping files described above. If yours can't you can instead write code to extract your config values. + Whenever QMK generates a complete `info.json` it extracts information from `config.h` and `rules.mk`. You will need to add code for your new config value to `lib/python/qmk/info.py`. Typically this means adding a new `_extract_()` function and then calling your function in either `_extract_config_h()` or `_extract_rules_mk()`. If you are not sure how to edit this file or are not comfortable with Python [open an issue](https://github.com/qmk/qmk_firmware/issues/new?assignees=&labels=cli%2C+python&template=other_issues.md&title=) or [join #cli on Discord](https://discord.gg/heQPAgy) and someone can help you with this part. diff --git a/keyboards/handwired/pytest/basic/info.json b/keyboards/handwired/pytest/basic/info.json new file mode 100644 index 0000000000..ed052a14a3 --- /dev/null +++ b/keyboards/handwired/pytest/basic/info.json @@ -0,0 +1,10 @@ +{ + "maintainer": "qmk", + "layouts": { + "LAYOUT_custom": { + "layout": [ + { "label": "KC_Q", "matrix": [0, 0], "w": 1, "x": 0, "y": 0 } + ] + } + } +} diff --git a/keyboards/vision_division/keymaps/default/config.h b/keyboards/vision_division/keymaps/default/config.h index b8f22a3b5f..aa8fc62aa0 100644 --- a/keyboards/vision_division/keymaps/default/config.h +++ b/keyboards/vision_division/keymaps/default/config.h @@ -8,7 +8,6 @@ #define VENDOR_ID 0xFEED #define DEVICE_VER 0x0001 #define MANUFACTURER IBNobody -#define PRODUCT Vision Division #define MATRIX_ROWS 6 #define MATRIX_ROW_PINS { C2, C3, F4, F5, F6, F7 } diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py index 70f32911a4..28d480707c 100755 --- a/lib/python/qmk/cli/doctor.py +++ b/lib/python/qmk/cli/doctor.py @@ -107,9 +107,9 @@ def doctor(cli): submodules.update() sub_ok = check_submodules() - if CheckStatus.ERROR in sub_ok: + if sub_ok == CheckStatus.ERROR: status = CheckStatus.ERROR - elif CheckStatus.WARNING in sub_ok and status == CheckStatus.OK: + elif sub_ok == CheckStatus.WARNING and status == CheckStatus.OK: status = CheckStatus.WARNING # Report a summary of our findings to the user diff --git a/lib/python/qmk/cli/generate/config_h.py b/lib/python/qmk/cli/generate/config_h.py index 1de84de7a9..7ddad745d1 100755 --- a/lib/python/qmk/cli/generate/config_h.py +++ b/lib/python/qmk/cli/generate/config_h.py @@ -1,62 +1,14 @@ """Used by the make system to generate info_config.h from info.json. """ +from pathlib import Path + +from dotty_dict import dotty from milc import cli -from qmk.constants import LED_INDICATORS from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.info import info_json, rgblight_animations, rgblight_properties, rgblight_toggles +from qmk.info import _json_load, info_json from qmk.path import is_keyboard, normpath -usb_prop_map = { - 'vid': 'VENDOR_ID', - 'pid': 'PRODUCT_ID', - 'device_ver': 'DEVICE_VER', -} - - -def debounce(debounce): - """Return the config.h lines that set debounce - """ - return """ -#ifndef DEBOUNCE -# define DEBOUNCE %s -#endif // DEBOUNCE -""" % debounce - - -def diode_direction(diode_direction): - """Return the config.h lines that set diode direction - """ - return """ -#ifndef DIODE_DIRECTION -# define DIODE_DIRECTION %s -#endif // DIODE_DIRECTION -""" % diode_direction - - -def keyboard_name(keyboard_name): - """Return the config.h lines that set the keyboard's name. - """ - return """ -#ifndef DESCRIPTION -# define DESCRIPTION %s -#endif // DESCRIPTION - -#ifndef PRODUCT -# define PRODUCT %s -#endif // PRODUCT -""" % (keyboard_name.replace("'", ""), keyboard_name.replace("'", "")) - - -def manufacturer(manufacturer): - """Return the config.h lines that set the manufacturer. - """ - return """ -#ifndef MANUFACTURER -# define MANUFACTURER %s -#endif // MANUFACTURER -""" % (manufacturer.replace("'", "")) - def direct_pins(direct_pins): """Return the config.h lines that set the direct pins. @@ -72,80 +24,34 @@ def direct_pins(direct_pins): return """ #ifndef MATRIX_COLS -# define MATRIX_COLS %s +# define MATRIX_COLS %s #endif // MATRIX_COLS #ifndef MATRIX_ROWS -# define MATRIX_ROWS %s +# define MATRIX_ROWS %s #endif // MATRIX_ROWS #ifndef DIRECT_PINS -# define DIRECT_PINS {%s} +# define DIRECT_PINS {%s} #endif // DIRECT_PINS """ % (col_count, row_count, ','.join(rows)) -def col_pins(col_pins): - """Return the config.h lines that set the column pins. +def pin_array(define, pins): + """Return the config.h lines that set a pin array. """ - cols = ','.join(map(str, [pin or 'NO_PIN' for pin in col_pins])) - col_num = len(col_pins) + pin_num = len(pins) + pin_array = ', '.join(map(str, [pin or 'NO_PIN' for pin in pins])) - return """ -#ifndef MATRIX_COLS -# define MATRIX_COLS %s -#endif // MATRIX_COLS + return f""" +#ifndef {define}S +# define {define}S {pin_num} +#endif // {define}S -#ifndef MATRIX_COL_PINS -# define MATRIX_COL_PINS {%s} -#endif // MATRIX_COL_PINS -""" % (col_num, cols) - - -def row_pins(row_pins): - """Return the config.h lines that set the row pins. - """ - rows = ','.join(map(str, [pin or 'NO_PIN' for pin in row_pins])) - row_num = len(row_pins) - - return """ -#ifndef MATRIX_ROWS -# define MATRIX_ROWS %s -#endif // MATRIX_ROWS - -#ifndef MATRIX_ROW_PINS -# define MATRIX_ROW_PINS {%s} -#endif // MATRIX_ROW_PINS -""" % (row_num, rows) - - -def indicators(config): - """Return the config.h lines that setup LED indicators. - """ - defines = [] - - for led, define in LED_INDICATORS.items(): - if led in config: - defines.append('') - defines.append('#ifndef %s' % (define,)) - defines.append('# define %s %s' % (define, config[led])) - defines.append('#endif // %s' % (define,)) - - return '\n'.join(defines) - - -def layout_aliases(layout_aliases): - """Return the config.h lines that setup layout aliases. - """ - defines = [] - - for alias, layout in layout_aliases.items(): - defines.append('') - defines.append('#ifndef %s' % (alias,)) - defines.append('# define %s %s' % (alias, layout)) - defines.append('#endif // %s' % (alias,)) - - return '\n'.join(defines) +#ifndef {define}_PINS +# define {define}_PINS {{ {pin_array} }} +#endif // {define}_PINS +""" def matrix_pins(matrix_pins): @@ -157,58 +63,14 @@ def matrix_pins(matrix_pins): pins.append(direct_pins(matrix_pins['direct'])) if 'cols' in matrix_pins: - pins.append(col_pins(matrix_pins['cols'])) + pins.append(pin_array('MATRIX_COL', matrix_pins['cols'])) if 'rows' in matrix_pins: - pins.append(row_pins(matrix_pins['rows'])) + pins.append(pin_array('MATRIX_ROW', matrix_pins['rows'])) return '\n'.join(pins) -def rgblight(config): - """Return the config.h lines that setup rgblight. - """ - rgblight_config = [] - - for json_key, config_key in rgblight_properties.items(): - if json_key in config: - rgblight_config.append('') - rgblight_config.append('#ifndef %s' % (config_key[0],)) - rgblight_config.append('# define %s %s' % (config_key[0], config[json_key])) - rgblight_config.append('#endif // %s' % (config_key[0],)) - - for json_key, config_key in rgblight_toggles.items(): - if config.get(json_key): - rgblight_config.append('') - rgblight_config.append('#ifndef %s' % (config_key,)) - rgblight_config.append('# define %s' % (config_key,)) - rgblight_config.append('#endif // %s' % (config_key,)) - - for json_key, config_key in rgblight_animations.items(): - if 'animations' in config and config['animations'].get(json_key): - rgblight_config.append('') - rgblight_config.append('#ifndef %s' % (config_key,)) - rgblight_config.append('# define %s' % (config_key,)) - rgblight_config.append('#endif // %s' % (config_key,)) - - return '\n'.join(rgblight_config) - - -def usb_properties(usb_props): - """Return the config.h lines that setup USB params. - """ - usb_lines = [] - - for info_name, config_name in usb_prop_map.items(): - if info_name in usb_props: - usb_lines.append('') - usb_lines.append('#ifndef ' + config_name) - usb_lines.append('# define %s %s' % (config_name, usb_props[info_name])) - usb_lines.append('#endif // ' + config_name) - - return '\n'.join(usb_lines) - - @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") @cli.argument('-kb', '--keyboard', help='Keyboard to generate config.h for.') @@ -228,39 +90,52 @@ def generate_config_h(cli): cli.log.error('Invalid keyboard: "%s"', cli.config.generate_config_h.keyboard) return False - # Build the info.json file - kb_info_json = info_json(cli.config.generate_config_h.keyboard) - # Build the info_config.h file. + kb_info_json = dotty(info_json(cli.config.generate_config_h.keyboard)) + info_config_map = _json_load(Path('data/mappings/info_config.json')) + config_h_lines = ['/* This file was generated by `qmk generate-config-h`. Do not edit or copy.' ' */', '', '#pragma once'] - if 'debounce' in kb_info_json: - config_h_lines.append(debounce(kb_info_json['debounce'])) + # Iterate through the info_config map to generate basic things + for config_key, info_dict in info_config_map.items(): + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + to_config = info_dict.get('to_config', True) - if 'diode_direction' in kb_info_json: - config_h_lines.append(diode_direction(kb_info_json['diode_direction'])) + if not to_config: + continue - if 'indicators' in kb_info_json: - config_h_lines.append(indicators(kb_info_json['indicators'])) + try: + config_value = kb_info_json[info_key] + except KeyError: + continue - if 'keyboard_name' in kb_info_json: - config_h_lines.append(keyboard_name(kb_info_json['keyboard_name'])) - - if 'layout_aliases' in kb_info_json: - config_h_lines.append(layout_aliases(kb_info_json['layout_aliases'])) - - if 'manufacturer' in kb_info_json: - config_h_lines.append(manufacturer(kb_info_json['manufacturer'])) - - if 'rgblight' in kb_info_json: - config_h_lines.append(rgblight(kb_info_json['rgblight'])) + if key_type.startswith('array'): + config_h_lines.append('') + config_h_lines.append(f'#ifndef {config_key}') + config_h_lines.append(f'# define {config_key} {{ {", ".join(map(str, config_value))} }}') + config_h_lines.append(f'#endif // {config_key}') + elif key_type == 'bool': + if config_value: + config_h_lines.append('') + config_h_lines.append(f'#ifndef {config_key}') + config_h_lines.append(f'# define {config_key}') + config_h_lines.append(f'#endif // {config_key}') + elif key_type == 'mapping': + for key, value in config_value.items(): + config_h_lines.append('') + config_h_lines.append(f'#ifndef {key}') + config_h_lines.append(f'# define {key} {value}') + config_h_lines.append(f'#endif // {key}') + else: + config_h_lines.append('') + config_h_lines.append(f'#ifndef {config_key}') + config_h_lines.append(f'# define {config_key} {config_value}') + config_h_lines.append(f'#endif // {config_key}') if 'matrix_pins' in kb_info_json: config_h_lines.append(matrix_pins(kb_info_json['matrix_pins'])) - if 'usb' in kb_info_json: - config_h_lines.append(usb_properties(kb_info_json['usb'])) - # Show the results config_h = '\n'.join(config_h_lines) diff --git a/lib/python/qmk/cli/generate/rules_mk.py b/lib/python/qmk/cli/generate/rules_mk.py index b262e3c666..af740f341d 100755 --- a/lib/python/qmk/cli/generate/rules_mk.py +++ b/lib/python/qmk/cli/generate/rules_mk.py @@ -1,16 +1,37 @@ """Used by the make system to generate a rules.mk """ +from pathlib import Path + +from dotty_dict import dotty from milc import cli from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.info import info_json +from qmk.info import _json_load, info_json from qmk.path import is_keyboard, normpath -info_to_rules = { - 'board': 'BOARD', - 'bootloader': 'BOOTLOADER', - 'processor': 'MCU', -} + +def process_mapping_rule(kb_info_json, rules_key, info_dict): + """Return the rules.mk line(s) for a mapping rule. + """ + if not info_dict.get('to_c', True): + return None + + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + + try: + rules_value = kb_info_json[info_key] + except KeyError: + return None + + if key_type == 'array': + return f'{rules_key} ?= {" ".join(rules_value)}' + elif key_type == 'bool': + return f'{rules_key} ?= {"on" if rules_value else "off"}' + elif key_type == 'mapping': + return '\n'.join([f'{key} ?= {value}' for key, value in rules_value.items()]) + + return f'{rules_key} ?= {rules_value}' @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') @@ -22,7 +43,6 @@ info_to_rules = { def generate_rules_mk(cli): """Generates a rules.mk file from info.json. """ - # Determine our keyboard(s) if not cli.config.generate_rules_mk.keyboard: cli.log.error('Missing paramater: --keyboard') cli.subcommands['info'].print_help() @@ -32,16 +52,18 @@ def generate_rules_mk(cli): cli.log.error('Invalid keyboard: "%s"', cli.config.generate_rules_mk.keyboard) return False - # Build the info.json file - kb_info_json = info_json(cli.config.generate_rules_mk.keyboard) + kb_info_json = dotty(info_json(cli.config.generate_rules_mk.keyboard)) + info_rules_map = _json_load(Path('data/mappings/info_rules.json')) rules_mk_lines = ['# This file was generated by `qmk generate-rules-mk`. Do not edit or copy.', ''] - # Bring in settings - for info_key, rule_key in info_to_rules.items(): - if info_key in kb_info_json: - rules_mk_lines.append(f'{rule_key} ?= {kb_info_json[info_key]}') + # Iterate through the info_rules map to generate basic rules + for rules_key, info_dict in info_rules_map.items(): + new_entry = process_mapping_rule(kb_info_json, rules_key, info_dict) - # Find features that should be enabled + if new_entry: + rules_mk_lines.append(new_entry) + + # Iterate through features to enable/disable them if 'features' in kb_info_json: for feature, enabled in kb_info_json['features'].items(): if feature == 'bootmagic_lite' and enabled: @@ -51,15 +73,6 @@ def generate_rules_mk(cli): enabled = 'yes' if enabled else 'no' rules_mk_lines.append(f'{feature}_ENABLE ?= {enabled}') - # Set the LED driver - if 'led_matrix' in kb_info_json and 'driver' in kb_info_json['led_matrix']: - driver = kb_info_json['led_matrix']['driver'] - rules_mk_lines.append(f'LED_MATRIX_DRIVER ?= {driver}') - - # Add community layouts - if 'community_layouts' in kb_info_json: - rules_mk_lines.append(f'LAYOUTS ?= {" ".join(kb_info_json["community_layouts"])}') - # Show the results rules_mk = '\n'.join(rules_mk_lines) + '\n' @@ -72,7 +85,7 @@ def generate_rules_mk(cli): if cli.args.quiet: print(cli.args.output) else: - cli.log.info('Wrote info_config.h to %s.', cli.args.output) + cli.log.info('Wrote rules.mk to %s.', cli.args.output) else: print(rules_mk) diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index cb94613562..8a177eb6b5 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -27,7 +27,7 @@ ROW_LETTERS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnop' LED_INDICATORS = { 'caps_lock': 'LED_CAPS_LOCK_PIN', 'num_lock': 'LED_NUM_LOCK_PIN', - 'scrol_lock': 'LED_SCROLL_LOCK_PIN', + 'scroll_lock': 'LED_SCROLL_LOCK_PIN', } # Constants that should match their counterparts in make diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index cc81f7a086..2accaba9e4 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -5,57 +5,18 @@ from collections.abc import Mapping from glob import glob from pathlib import Path +import hjson import jsonschema +from dotty_dict import dotty from milc import cli -from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS, LED_INDICATORS +from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS from qmk.c_parse import find_layouts from qmk.keyboard import config_h, rules_mk from qmk.keymap import list_keymaps from qmk.makefile import parse_rules_mk_file from qmk.math import compute -led_matrix_properties = { - 'driver_count': 'LED_DRIVER_COUNT', - 'driver_addr1': 'LED_DRIVER_ADDR_1', - 'driver_addr2': 'LED_DRIVER_ADDR_2', - 'driver_addr3': 'LED_DRIVER_ADDR_3', - 'driver_addr4': 'LED_DRIVER_ADDR_4', - 'led_count': 'LED_DRIVER_LED_COUNT', - 'timeout': 'ISSI_TIMEOUT', - 'persistence': 'ISSI_PERSISTENCE' -} - -rgblight_properties = { - 'led_count': ('RGBLED_NUM', int), - 'pin': ('RGB_DI_PIN', str), - 'max_brightness': ('RGBLIGHT_LIMIT_VAL', int), - 'hue_steps': ('RGBLIGHT_HUE_STEP', int), - 'saturation_steps': ('RGBLIGHT_SAT_STEP', int), - 'brightness_steps': ('RGBLIGHT_VAL_STEP', int) -} - -rgblight_toggles = { - 'sleep': 'RGBLIGHT_SLEEP', - 'split': 'RGBLIGHT_SPLIT', -} - -rgblight_animations = { - 'all': 'RGBLIGHT_ANIMATIONS', - 'alternating': 'RGBLIGHT_EFFECT_ALTERNATING', - 'breathing': 'RGBLIGHT_EFFECT_BREATHING', - 'christmas': 'RGBLIGHT_EFFECT_CHRISTMAS', - 'knight': 'RGBLIGHT_EFFECT_KNIGHT', - 'rainbow_mood': 'RGBLIGHT_EFFECT_RAINBOW_MOOD', - 'rainbow_swirl': 'RGBLIGHT_EFFECT_RAINBOW_SWIRL', - 'rgb_test': 'RGBLIGHT_EFFECT_RGB_TEST', - 'snake': 'RGBLIGHT_EFFECT_SNAKE', - 'static_gradient': 'RGBLIGHT_EFFECT_STATIC_GRADIENT', - 'twinkle': 'RGBLIGHT_EFFECT_TWINKLE' -} - -usb_properties = {'vid': 'VENDOR_ID', 'pid': 'PRODUCT_ID', 'device_ver': 'DEVICE_VER'} - true_values = ['1', 'on', 'yes'] false_values = ['0', 'off', 'no'] @@ -101,7 +62,6 @@ def info_json(keyboard): except jsonschema.ValidationError as e: json_path = '.'.join([str(p) for p in e.absolute_path]) cli.log.error('Invalid API data: %s: %s: %s', keyboard, json_path, e.message) - print(dir(e)) exit() # Make sure we have at least one layout @@ -132,7 +92,7 @@ def _json_load(json_file): Note: file must be a Path object. """ try: - return json.load(json_file.open()) + return hjson.load(json_file.open()) except json.decoder.JSONDecodeError as e: cli.log.error('Invalid JSON encountered attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e) @@ -172,65 +132,6 @@ def keyboard_api_validate(data): return validator(data) -def _extract_debounce(info_data, config_c): - """Handle debounce. - """ - if 'debounce' in info_data and 'DEBOUNCE' in config_c: - _log_warning(info_data, 'Debounce is specified in both info.json and config.h, the config.h value wins.') - - if 'DEBOUNCE' in config_c: - info_data['debounce'] = int(config_c['DEBOUNCE']) - - return info_data - - -def _extract_diode_direction(info_data, config_c): - """Handle the diode direction. - """ - if 'diode_direction' in info_data and 'DIODE_DIRECTION' in config_c: - _log_warning(info_data, 'Diode direction is specified in both info.json and config.h, the config.h value wins.') - - if 'DIODE_DIRECTION' in config_c: - info_data['diode_direction'] = config_c.get('DIODE_DIRECTION') - - return info_data - - -def _extract_indicators(info_data, config_c): - """Find the LED indicator information. - """ - for json_key, config_key in LED_INDICATORS.items(): - if json_key in info_data.get('indicators', []) and config_key in config_c: - _log_warning(info_data, f'Indicator {json_key} is specified in both info.json and config.h, the config.h value wins.') - - if 'indicators' not in info_data: - info_data['indicators'] = {} - - if config_key in config_c: - if 'indicators' not in info_data: - info_data['indicators'] = {} - - info_data['indicators'][json_key] = config_c.get(config_key) - - return info_data - - -def _extract_community_layouts(info_data, rules): - """Find the community layouts in rules.mk. - """ - community_layouts = rules['LAYOUTS'].split() if 'LAYOUTS' in rules else [] - - if 'community_layouts' in info_data: - for layout in community_layouts: - if layout not in info_data['community_layouts']: - community_layouts.append(layout) - - else: - info_data['community_layouts'] = community_layouts - - return info_data - - def _extract_features(info_data, rules): """Find all the features enabled in rules.mk. """ @@ -267,78 +168,6 @@ def _extract_features(info_data, rules): return info_data -def _extract_led_drivers(info_data, rules): - """Find all the LED drivers set in rules.mk. - """ - if 'LED_MATRIX_DRIVER' in rules: - if 'led_matrix' not in info_data: - info_data['led_matrix'] = {} - - if info_data['led_matrix'].get('driver'): - _log_warning(info_data, 'LED Matrix driver is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['led_matrix']['driver'] = rules['LED_MATRIX_DRIVER'] - - return info_data - - -def _extract_led_matrix(info_data, config_c): - """Handle the led_matrix configuration. - """ - led_matrix = info_data.get('led_matrix', {}) - - for json_key, config_key in led_matrix_properties.items(): - if config_key in config_c: - if json_key in led_matrix: - _log_warning(info_data, 'LED Matrix: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,)) - - led_matrix[json_key] = config_c[config_key] - - -def _extract_rgblight(info_data, config_c): - """Handle the rgblight configuration. - """ - rgblight = info_data.get('rgblight', {}) - animations = rgblight.get('animations', {}) - - if 'RGBLED_SPLIT' in config_c: - raw_split = config_c.get('RGBLED_SPLIT', '').replace('{', '').replace('}', '').strip() - rgblight['split_count'] = [int(i) for i in raw_split.split(',')] - - for json_key, config_key_type in rgblight_properties.items(): - config_key, config_type = config_key_type - - if config_key in config_c: - if json_key in rgblight: - _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,)) - - try: - rgblight[json_key] = config_type(config_c[config_key]) - except ValueError as e: - cli.log.error('%s: config.h: Could not convert "%s" to %s: %s', info_data['keyboard_folder'], config_c[config_key], config_type.__name__, e) - - for json_key, config_key in rgblight_toggles.items(): - if config_key in config_c and json_key in rgblight: - _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.', json_key) - - rgblight[json_key] = config_key in config_c - - for json_key, config_key in rgblight_animations.items(): - if config_key in config_c: - if json_key in animations: - _log_warning(info_data, 'RGB Light: animations: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,)) - - animations[json_key] = config_c[config_key] - - if animations: - rgblight['animations'] = animations - - if rgblight: - info_data['rgblight'] = rgblight - - return info_data - - def _pin_name(pin): """Returns the proper representation for a pin. """ @@ -426,34 +255,59 @@ def _extract_matrix_info(info_data, config_c): return info_data -def _extract_usb_info(info_data, config_c): - """Populate the USB information. - """ - if 'usb' not in info_data: - info_data['usb'] = {} - - for info_name, config_name in usb_properties.items(): - if config_name in config_c: - if info_name in info_data['usb']: - _log_warning(info_data, '%s in config.h is overwriting usb.%s in info.json' % (config_name, info_name)) - - info_data['usb'][info_name] = '0x' + config_c[config_name][2:].upper() - - return info_data - - def _extract_config_h(info_data): """Pull some keyboard information from existing config.h files """ config_c = config_h(info_data['keyboard_folder']) - _extract_debounce(info_data, config_c) - _extract_diode_direction(info_data, config_c) - _extract_indicators(info_data, config_c) + # Pull in data from the json map + dotty_info = dotty(info_data) + info_config_map = _json_load(Path('data/mappings/info_config.json')) + + for config_key, info_dict in info_config_map.items(): + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + + try: + if config_key in config_c and info_dict.get('to_json', True): + if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): + _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key)) + + if key_type.startswith('array'): + if '.' in key_type: + key_type, array_type = key_type.split('.', 1) + else: + array_type = None + + config_value = config_c[config_key].replace('{', '').replace('}', '').strip() + + if array_type == 'int': + dotty_info[info_key] = list(map(int, config_value.split(','))) + else: + dotty_info[info_key] = config_value.split(',') + + elif key_type == 'bool': + dotty_info[info_key] = config_c[config_key] in true_values + + elif key_type == 'hex': + dotty_info[info_key] = '0x' + config_c[config_key][2:].upper() + + elif key_type == 'list': + dotty_info[info_key] = config_c[config_key].split() + + elif key_type == 'int': + dotty_info[info_key] = int(config_c[config_key]) + + else: + dotty_info[info_key] = config_c[config_key] + + except Exception as e: + _log_warning(info_data, f'{config_key}->{info_key}: {e}') + + info_data.update(dotty_info) + + # Pull data that easily can't be mapped in json _extract_matrix_info(info_data, config_c) - _extract_usb_info(info_data, config_c) - _extract_led_matrix(info_data, config_c) - _extract_rgblight(info_data, config_c) return info_data @@ -462,21 +316,66 @@ def _extract_rules_mk(info_data): """Pull some keyboard information from existing rules.mk files """ rules = rules_mk(info_data['keyboard_folder']) - mcu = rules.get('MCU', info_data.get('processor')) + info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4')) - if mcu in CHIBIOS_PROCESSORS: + if info_data['processor'] in CHIBIOS_PROCESSORS: arm_processor_rules(info_data, rules) - elif mcu in LUFA_PROCESSORS + VUSB_PROCESSORS: + elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS: avr_processor_rules(info_data, rules) else: - cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], mcu)) + cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor'])) unknown_processor_rules(info_data, rules) - _extract_community_layouts(info_data, rules) + # Pull in data from the json map + dotty_info = dotty(info_data) + info_rules_map = _json_load(Path('data/mappings/info_rules.json')) + + for rules_key, info_dict in info_rules_map.items(): + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + + try: + if rules_key in rules and info_dict.get('to_json', True): + if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): + _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key)) + + if key_type.startswith('array'): + if '.' in key_type: + key_type, array_type = key_type.split('.', 1) + else: + array_type = None + + rules_value = rules[rules_key].replace('{', '').replace('}', '').strip() + + if array_type == 'int': + dotty_info[info_key] = list(map(int, rules_value.split(','))) + else: + dotty_info[info_key] = rules_value.split(',') + + elif key_type == 'list': + dotty_info[info_key] = rules[rules_key].split() + + elif key_type == 'bool': + dotty_info[info_key] = rules[rules_key] in true_values + + elif key_type == 'hex': + dotty_info[info_key] = '0x' + rules[rules_key][2:].upper() + + elif key_type == 'int': + dotty_info[info_key] = int(rules[rules_key]) + + else: + dotty_info[info_key] = rules[rules_key] + + except Exception as e: + _log_warning(info_data, f'{rules_key}->{info_key}: {e}') + + info_data.update(dotty_info) + + # Merge in config values that can't be easily mapped _extract_features(info_data, rules) - _extract_led_drivers(info_data, rules) return info_data @@ -565,23 +464,7 @@ def arm_processor_rules(info_data, rules): info_data['processor_type'] = 'arm' info_data['protocol'] = 'ChibiOS' - if 'MCU' in rules: - if 'processor' in info_data: - _log_warning(info_data, 'Processor/MCU is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['processor'] = rules['MCU'] - - elif 'processor' not in info_data: - info_data['processor'] = 'unknown' - - if 'BOOTLOADER' in rules: - # FIXME(skullydazed/anyone): need to remove the massive amounts of duplication first - # if 'bootloader' in info_data: - # _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['bootloader'] = rules['BOOTLOADER'] - - else: + if 'bootloader' not in info_data: if 'STM32' in info_data['processor']: info_data['bootloader'] = 'stm32-dfu' else: @@ -594,12 +477,6 @@ def arm_processor_rules(info_data, rules): elif 'ARM_ATSAM' in rules: info_data['platform'] = 'ARM_ATSAM' - if 'BOARD' in rules: - if 'board' in info_data: - _log_warning(info_data, 'Board is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['board'] = rules['BOARD'] - return info_data @@ -607,26 +484,10 @@ def avr_processor_rules(info_data, rules): """Setup the default info for an AVR board. """ info_data['processor_type'] = 'avr' - info_data['bootloader'] = rules['BOOTLOADER'] if 'BOOTLOADER' in rules else 'atmel-dfu' info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown' info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA' - if 'MCU' in rules: - if 'processor' in info_data: - _log_warning(info_data, 'Processor/MCU is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['processor'] = rules['MCU'] - - elif 'processor' not in info_data: - info_data['processor'] = 'unknown' - - if 'BOOTLOADER' in rules: - # FIXME(skullydazed/anyone): need to remove the massive amounts of duplication first - # if 'bootloader' in info_data: - # _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.') - - info_data['bootloader'] = rules['BOOTLOADER'] - else: + if 'bootloader' not in info_data: info_data['bootloader'] = 'atmel-dfu' # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk: diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py index f889833d0b..3efeddb85e 100644 --- a/lib/python/qmk/tests/test_cli_commands.py +++ b/lib/python/qmk/tests/test_cli_commands.py @@ -230,3 +230,32 @@ def test_generate_rgb_breathe_table(): check_returncode(result) assert 'Breathing center: 1.2' in result.stdout assert 'Breathing max: 127' in result.stdout + + +def test_generate_config_h(): + result = check_subcommand('generate-config-h', '-kb', 'handwired/pytest/basic') + check_returncode(result) + assert '# define DEVICE_VER 0x0001' in result.stdout + assert '# define DESCRIPTION handwired/pytest/basic' in result.stdout + assert '# define DIODE_DIRECTION COL2ROW' in result.stdout + assert '# define MANUFACTURER none' in result.stdout + assert '# define PRODUCT handwired/pytest/basic' in result.stdout + assert '# define PRODUCT_ID 0x6465' in result.stdout + assert '# define VENDOR_ID 0xFEED' in result.stdout + assert '# define MATRIX_COLS 1' in result.stdout + assert '# define MATRIX_COL_PINS { F4 }' in result.stdout + assert '# define MATRIX_ROWS 1' in result.stdout + assert '# define MATRIX_ROW_PINS { F5 }' in result.stdout + + +def test_generate_rules_mk(): + result = check_subcommand('generate-rules-mk', '-kb', 'handwired/pytest/basic') + check_returncode(result) + assert 'BOOTLOADER ?= atmel-dfu' in result.stdout + assert 'MCU ?= atmega32u4' in result.stdout + + +def test_generate_layouts(): + result = check_subcommand('generate-layouts', '-kb', 'handwired/pytest/basic') + check_returncode(result) + assert '#define LAYOUT_custom(k0A) {' in result.stdout diff --git a/requirements.txt b/requirements.txt index f4d43da8d6..27a6baed99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ appdirs argcomplete colorama +dotty-dict hjson jsonschema milc