Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
608 changes: 608 additions & 0 deletions Buildscripts/TactilitySDK/Tools/_remote_viewer.py

Large diffs are not rendered by default.

36 changes: 36 additions & 0 deletions Buildscripts/TactilitySDK/Tools/remote.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
# License: Apache License v2
"""Shows the display of the device and sends keyboard, touch and mouse input to it. Run it as "tt remote"."""

import argparse
import os

import shell
import _remote_viewer

# How long the device may take to start the session after it accepted the connection
START_TIMEOUT_SECONDS = 10


def main():
shell.PROGRAM_NAME = "tt remote" if "TACTILITY_SDK_VERSION" in os.environ else "remote.py"
parser = argparse.ArgumentParser(prog=shell.PROGRAM_NAME, description="show the display of the device and control it", add_help=False)
shell.add_connection_arguments(parser)
args = shell.parse_arguments(parser)

connection = shell.open_session(args.host, args.port, "remote")
frame_type, payload = connection.read_frame(START_TIMEOUT_SECONDS)
if frame_type == shell.FRAME_FAIL:
shell.fail(payload.decode("utf-8", errors="replace"))
state = _remote_viewer.parse_state(payload) if frame_type == shell.FRAME_REMOTE_INFO else None
if state is None:
shell.fail("the device did not describe its display")

session = _remote_viewer.RemoteSession(connection)
_remote_viewer.run_viewer(session, state, f"{shell.PROGRAM_NAME} {args.host}")


if __name__ == "__main__":
import _bootstrap
_bootstrap.run()
main()
6 changes: 6 additions & 0 deletions Buildscripts/TactilitySDK/Tools/shell.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@
FRAME_APP_INSTALL = 0x20
FRAME_APP_UNINSTALL = 0x21
FRAME_APP_RUN = 0x22
FRAME_REMOTE_INFO = 0x30
FRAME_REMOTE_RECT = 0x31
FRAME_REMOTE_STATE = 0x32
FRAME_INPUT_ENABLE = 0x40
FRAME_INPUT_KEY = 0x41
FRAME_INPUT_TOUCH = 0x42
SYNC_FILE_MISSING = 0
SYNC_FILE_DIRECTORY = 2
SYNC_READ_SIZE = 64 * 1024
Expand Down
28 changes: 21 additions & 7 deletions Buildscripts/TactilitySDK/Tools/ui.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Tactility Dev Console — a CustomTkinter desktop version of the SDK's `tt ui` dashboard.
"""Tactility Dev Console — a Custom2kinter desktop version of the SDK's `tt ui` dashboard.

Pick an app, choose a target, and run build / install / run / clean with live colored
output. All of the actual work is delegated to the repo's own tt.py, so SDK selection,
Expand Down Expand Up @@ -39,10 +39,17 @@

try:
import customtkinter as ctk
except ImportError: # pragma: no cover - dependency hint
except ImportError as _error: # pragma: no cover - dependency hint
raise SystemExit(
"CustomTkinter is required for this tool.\n"
"Install it with: python -m pip install customtkinter"
f"Custom2kinter can't be imported by {sys.executable}: {_error}\n"
+ (
"Pillow has no Tk support (PIL.ImageTk) in that Python. Install it with your package manager:\n"
" Fedora: sudo dnf install python3-pillow-tk\n"
" Debian/Ubuntu: sudo apt install python3-pil.imagetk\n"
"(The 'ImageTk' package on PyPI is something else.)"
if "ImageTk" in str(_error)
else f"Install it for that Python with: {sys.executable} -m pip install custom2kinter"
)
) from None

try:
Expand Down Expand Up @@ -1025,7 +1032,10 @@ def _build_ui(self) -> None:
self.status_label = ctk.CTkLabel(bottom, text="Idle", anchor="w",
text_color=("gray40", "gray60"))
self.status_label.grid(row=0, column=0, sticky="w")
self.progress = ctk.CTkProgressBar(bottom, width=180, mode="indeterminate")
try:
self.progress = ctk.CTkProgressBar(bottom, length=180, mode="indeterminate")
except Exception: # CustomTkinter 5.2.x has no `length` option.
self.progress = ctk.CTkProgressBar(bottom, width=180, mode="indeterminate")
self.progress.grid(row=0, column=1, sticky="e")
self.progress.set(0)
self.progress.grid_remove()
Expand Down Expand Up @@ -1446,8 +1456,12 @@ def _show_screenshot(self, path: Path) -> None:
with Image.open(path) as source:
source.thumbnail((900, 160))
width, height = source.size
image = ctk.CTkImage(light_image=source.copy(), dark_image=source.copy(),
size=(width, height))
try:
image = ctk.CTkImage(light_image=source.copy(), dark_image=source.copy(),
width=width, height=height)
except TypeError:
image = ctk.CTkImage(light_image=source.copy(), dark_image=source.copy(),
size=(width, height))
except (OSError, ValueError) as exc:
self.screenshot_preview.configure(text="Preview unavailable", image=None, height=120)
self.screenshot_info.configure(text=f"Captured {path} (preview unavailable: {exc})")
Expand Down
2 changes: 1 addition & 1 deletion Devices/cl32-v04/source/cl32_v4_keyboard.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -290,9 +290,9 @@ static error_t v4_read_key(Device* device, KeyboardKeyData* data) {
}

static constexpr KeyboardApi cl32_v4_keyboard_api = {
.capabilities = 0,
.read_key = v4_read_key,
.get_backlight = nullptr,
.is_present = nullptr,
};

static error_t start(Device* device) {
Expand Down
2 changes: 1 addition & 1 deletion Devices/m5stack-tab5/Source/devices/tab5_keyboard.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -656,9 +656,9 @@ static error_t tab5_keyboard_read_key(Device* device, KeyboardKeyData* data) {
// endregion

static const KeyboardApi tab5_keyboard_api = {
.capabilities = 0,
.read_key = tab5_keyboard_read_key,
.get_backlight = nullptr,
.is_present = nullptr,
};

// Defined in module.cpp - this driver is registered directly by m5stack-tab5's own module,
Expand Down
2 changes: 1 addition & 1 deletion Devices/simulator/Source/drivers/sdl_keyboard.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ static error_t sdl_keyboard_read_key(Device*, KeyboardKeyData* data) {
// endregion

static const KeyboardApi sdl_keyboard_api = {
.capabilities = 0,
.read_key = sdl_keyboard_read_key,
.get_backlight = nullptr,
.is_present = nullptr,
};

extern Module simulator_module;
Expand Down
7 changes: 1 addition & 6 deletions Drivers/button-control-module/source/button_control.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -200,17 +200,12 @@ static error_t button_control_read_key(Device* device, KeyboardKeyData* data) {
return ERROR_NONE;
}

// Buttons can't type text, so they don't count as a hardware keyboard and the software keyboard stays available
static bool button_control_is_present(Device*) {
return false;
}

// endregion

static constexpr KeyboardApi button_control_api = {
.capabilities = KEYBOARD_CAPABILITY_RESTRICTED,
.read_key = button_control_read_key,
.get_backlight = nullptr,
.is_present = button_control_is_present,
};

Driver button_control_driver = {
Expand Down
2 changes: 1 addition & 1 deletion Drivers/gpio-encoder-module/source/gpio_encoder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -318,9 +318,9 @@ static error_t gpio_encoder_read_key(Device* device, KeyboardKeyData* data) {
// endregion

static constexpr KeyboardApi GPIO_ENCODER_API = {
.capabilities = KEYBOARD_CAPABILITY_RESTRICTED,
.read_key = gpio_encoder_read_key,
.get_backlight = nullptr,
.is_present = nullptr,
};

Driver gpio_encoder_driver = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ static error_t gpio_trackball_keyboard_read_key(Device* device, KeyboardKeyData*
}

static const KeyboardApi GPIO_TRACKBALL_KEYBOARD_API = {
.capabilities = KEYBOARD_CAPABILITY_RESTRICTED,
.read_key = gpio_trackball_keyboard_read_key,
.get_backlight = nullptr,
.is_present = nullptr,
};

// endregion
Expand Down
2 changes: 1 addition & 1 deletion Drivers/lilygo-module/source/tdeck_keyboard.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,9 @@ static error_t tdeck_keyboard_read_key(Device* device, KeyboardKeyData* data) {
// endregion

static const KeyboardApi tdeck_keyboard_api = {
.capabilities = 0,
.read_key = tdeck_keyboard_read_key,
.get_backlight = nullptr,
.is_present = nullptr,
};

Driver tdeck_keyboard_driver = {
Expand Down
2 changes: 1 addition & 1 deletion Drivers/m5stack-module/source/cardputer_adv_keyboard.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -265,9 +265,9 @@ static error_t cardputer_adv_keyboard_read_key(Device* device, KeyboardKeyData*
// endregion

static const KeyboardApi cardputer_adv_keyboard_api = {
.capabilities = 0,
.read_key = cardputer_adv_keyboard_read_key,
.get_backlight = nullptr,
.is_present = nullptr,
};

Driver cardputer_adv_keyboard_driver = {
Expand Down
2 changes: 1 addition & 1 deletion Drivers/m5stack-module/source/cardputer_keyboard.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -296,9 +296,9 @@ static error_t cardputer_keyboard_read_key(Device* device, KeyboardKeyData* data
// endregion

static const KeyboardApi cardputer_keyboard_api = {
.capabilities = 0,
.read_key = cardputer_keyboard_read_key,
.get_backlight = nullptr,
.is_present = nullptr,
};

Driver cardputer_keyboard_driver = {
Expand Down
16 changes: 8 additions & 8 deletions Drivers/rgb-display-module/source/rgb_display.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -454,14 +454,14 @@ static error_t rgb_display_get_backlight(Device* device, Device** backlight) {
}

constexpr uint32_t RGB_DISPLAY_CAPABILITIES = DISPLAY_CAPABILITY_CAP_MIRROR | DISPLAY_CAPABILITY_CAP_SWAP_XY |
DISPLAY_CAPABILITY_INVERT_COLOR | DISPLAY_CAPABILITY_ON_OFF | DISPLAY_CAPABILITY_BACKLIGHT;

// esp_lcd_panel_rgb only applies mirror()/swap_xy()'s rotate_mask when draw_bitmap()'s color_data
// is copied into the frame buffer by CPU. When frame_buffer_count > 0, LVGL is bound directly onto
// the panel's own frame buffers (see lvgl_display.c), so color_data always already *is* the frame
// buffer and that copy - and with it the rotation - never happens. Report those two capabilities as
// unavailable in that configuration so callers (e.g. lvgl_display.c) don't rely on a rotation that
// silently does nothing.
DISPLAY_CAPABILITY_INVERT_COLOR | DISPLAY_CAPABILITY_ON_OFF | DISPLAY_CAPABILITY_BACKLIGHT |
DISPLAY_CAPABILITY_PREFER_EXTERNAL_RAM;

// Mirror and swap-xy are reported unavailable when the panel has frame buffers, because
// esp_lcd_panel_rgb only applies them when draw_bitmap() copies the data into a frame buffer.
// A caller that renders straight into a frame buffer would get no rotation. The LVGL bridge reads
// the missing capabilities as no hardware rotation, so it renders into its own draw buffer and
// rotates in software.
static bool rgb_display_has_capability(Device* device, uint32_t capability) {
auto* internal = static_cast<RgbDisplayInternal*>(device_get_driver_data(device));
uint32_t capabilities = RGB_DISPLAY_CAPABILITIES;
Expand Down
2 changes: 1 addition & 1 deletion Drivers/rotary-button-module/source/rotary_button.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -175,9 +175,9 @@ static error_t rotary_button_read_key(Device* device, KeyboardKeyData* data) {
// endregion

static constexpr KeyboardApi ROTARY_BUTTON_API = {
.capabilities = KEYBOARD_CAPABILITY_RESTRICTED,
.read_key = rotary_button_read_key,
.get_backlight = nullptr,
.is_present = nullptr,
};

Driver rotary_button_driver = {
Expand Down
2 changes: 1 addition & 1 deletion Drivers/tca8418-module/source/tca8418.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -273,9 +273,9 @@ static error_t tca8418_read_key(Device* device, KeyboardKeyData* data) {
// endregion

static constexpr KeyboardApi tca8418_api = {
.capabilities = 0,
.read_key = tca8418_read_key,
.get_backlight = nullptr,
.is_present = nullptr,
};

Driver tca8418_driver = {
Expand Down
2 changes: 1 addition & 1 deletion Modules/development-module/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ list(APPEND REQUIRES_LIST
)

if (DEFINED ENV{ESP_IDF_VERSION})
list(APPEND REQUIRES_LIST lwip)
list(APPEND REQUIRES_LIST lwip esp_rom)
endif ()

tactility_add_module(development-module
Expand Down
24 changes: 22 additions & 2 deletions Modules/development-module/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,30 @@ The C API is in `<development/server.h>`.
- `0x21` uninstall `[package id]`: answered with `0x15` ok, also when the package isn't installed, or `0x16` fail `[message]`
- `0x22` run `[app id]`: starts the app, or restarts it when it's in the foreground. Answered with `0x15` ok, or `0x16` fail `[message]`

7. In `info` mode, no client certificate is needed and the client isn't prompted for. The server answers with one JSON line and closes the connection:
7. In `remote` mode, the device streams its display until the client disconnects, and the client can send input. Only one remote session runs at a time.
The device needs external RAM, because it keeps the whole display image in memory. Without it, or without a display, the device answers `0x16` fail `[message]` after accepting.
The display is observed through `display_set_hook()`, so everything that draws through the kernel display API is streamed, also while LVGL is stopped. Content that was drawn before the session started only appears after the next draw.
Frames from the device can have up to 16384 bytes of payload. Frames from the client stay within 1024 bytes. All numbers are big-endian.
- `0x30` info and `0x32` state (device to client, 11 bytes): `[width u16][height u16][color format u8][swap xy u8][mirror x u8][mirror y u8][invert u8][on u8][rotation u8]`.
The device sends the info first and the state whenever one of these changes. The size is the size of the image that draw calls address, so it follows swap xy.
The orientation byte describes the UI, not the pixels: bits 0 and 1 are the rotation of the UI in quarter turns clockwise, and bit 2 is set when the display driver gets pixels that were rotated in software.
The client rotates the pixels clockwise by the rotation when bit 2 is set to show the UI upright, and the size stays that of the pixels that were drawn.
The touch device has the native orientation of the display, and the UI maps its positions by rotating them clockwise by the rotation, so the client converts touch positions back.
The color format is `DisplayColorFormat`: 1 BGR565, 2 BGR565 swapped, 3 RGB565, 4 RGB565 swapped, 5 RGB888, 6 grayscale. A monochrome display is sent as grayscale (0 black, 255 white), because the device unpacks its 1 bit pixels.
- `0x31` rectangle (device to client): `[x u16][y u16][width u16][height u16][flags u8][pixels]`. Bit 0 of the flags means that the pixels are a zlib stream.
The pixels are rows of the rectangle without padding in the color format of the display. A device without the compressor sends them raw.
- `0x40` enable input (client to device): `[mask u8]` with bit 0 keyboard and bit 1 touch. The device creates `remote_keyboard0` and `remote_touch0` for the bits in the mask
and removes the others. Answered with `0x15` ok, or `0x16` fail `[message]`. All of them are removed when the session ends.
- `0x41` key: `[codepoint u32][flags u8]` with bit 0 pressed, bit 1 ctrl and bit 2 alt. The codepoints are those of `KeyboardKeyData::key`.
- `0x42` touch: `[count u8]` followed by `count` points `[x u16][y u16]`, at most 5, in the native orientation of the display. The device passes them on as they are.
The frame lists all points that are touched at that moment and replaces the previous list, so a point that is no longer listed was released and a count of 0 releases everything.
The order of the points is not significant: the touch device reports them, and LVGL follows each finger by its position.
Input frames are ignored while their device doesn't exist.

8. In `info` mode, no client certificate is needed and the client isn't prompted for. The server answers with one JSON line and closes the connection:
`{"status":"accepted","info":{"vendor":"LilyGO","name":"T-Deck","id":"lilygo-tdeck","version":"0.8.0"}}`
The vendor is empty for devices without one.

`Buildscripts/TactilitySDK/Tools/shell.py` is a client for this server. App developers run it as `tt shell`, and copy files with `tt push` and `tt pull`.
`Buildscripts/TactilitySDK/Tools/shell.py` is a client for this server. App developers run it as `tt shell`, and copy files with `tt push` and `tt pull`. `tt remote` shows the display and sends input (`remote.py` and `_remote_viewer.py`).

License: [Apache v2.0](LICENSE-Apache-2.0.md)
45 changes: 45 additions & 0 deletions Modules/development-module/include/development/remote_access.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// SPDX-License-Identifier: Apache-2.0
#pragma once

#include <stdint.h>

#ifdef __cplusplus
extern "C" {
#endif

/** Bits of the value that DevelopmentRemoteUiHandlers::get_orientation returns */
#define DEVELOPMENT_REMOTE_ORIENTATION_ROTATION_MASK 0x03
/** The pixels that the display driver receives are rotated by the rotation, instead of the display doing that */
#define DEVELOPMENT_REMOTE_ORIENTATION_SOFTWARE 0x04

/**
* Functions of the UI that remote sessions use. Every member is optional.
*/
struct DevelopmentRemoteUiHandlers {
/** Makes the UI draw its whole screen, so that the first frames the viewer gets are complete. */
void (*redraw)(void);

/**
* @return the rotation of the UI in quarter turns clockwise (0 to 3), with DEVELOPMENT_REMOTE_ORIENTATION_SOFTWARE
* set when the display driver gets pixels that were rotated in software. Called regularly, as the UI can rotate at any time.
* The viewer uses it to show the UI upright and to convert touch positions.
*/
uint8_t (*get_orientation)(void);
};

/**
* Installs the display hook that tracks the display state for remote sessions. Call it before the
* UI configures a display, because a session can't ask a driver for state that was set later.
* Safe to call more than once. The module installs it too when it starts.
*/
void development_remote_start(void);

/** Removes the display hook */
void development_remote_stop(void);

/** Sets the handlers of the UI. @param[in] handlers must stay valid, or NULL to remove them */
void development_remote_set_ui_handlers(const struct DevelopmentRemoteUiHandlers* handlers);

#ifdef __cplusplus
}
#endif
20 changes: 20 additions & 0 deletions Modules/development-module/private/development/protocol.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,28 @@ enum DevelopmentFrameType : uint8_t {
DEVELOPMENT_FRAME_APP_INSTALL = 0x20,
DEVELOPMENT_FRAME_APP_UNINSTALL = 0x21,
DEVELOPMENT_FRAME_APP_RUN = 0x22,
// Remote mode, from the device
DEVELOPMENT_FRAME_REMOTE_INFO = 0x30,
DEVELOPMENT_FRAME_REMOTE_RECT = 0x31,
DEVELOPMENT_FRAME_REMOTE_STATE = 0x32,
// Remote mode, from the viewer
DEVELOPMENT_FRAME_INPUT_ENABLE = 0x40,
DEVELOPMENT_FRAME_INPUT_KEY = 0x41,
DEVELOPMENT_FRAME_INPUT_TOUCH = 0x42,
};

/** Bits of the DEVELOPMENT_FRAME_INPUT_ENABLE payload */
enum DevelopmentRemoteInput : uint8_t {
DEVELOPMENT_REMOTE_INPUT_KEYBOARD = 1 << 0,
DEVELOPMENT_REMOTE_INPUT_TOUCH = 1 << 1,
};

/**
* Remote mode frames from the device can be larger than DEVELOPMENT_FRAME_PAYLOAD_MAX.
* Frames from the viewer always stay within DEVELOPMENT_FRAME_PAYLOAD_MAX.
*/
#define DEVELOPMENT_REMOTE_FRAME_PAYLOAD_MAX 16384

enum DevelopmentSyncFileType : uint8_t {
DEVELOPMENT_SYNC_FILE_MISSING = 0,
DEVELOPMENT_SYNC_FILE_REGULAR = 1,
Expand Down
16 changes: 16 additions & 0 deletions Modules/development-module/private/development/remote.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// SPDX-License-Identifier: Apache-2.0
#pragma once

#include <development/session.h>

#include <app/event.h>

#include <tactility/concurrent/task_event_group.h>

/**
* Streams the display to the client and applies its input until it disconnects or the app is
* asked to close. Sends a DEVELOPMENT_FRAME_FAIL with the reason when remote access isn't possible.
* @param[in,out] close_requested set when an APP_EVENT_CLOSE arrives
*/
void development_remote_run(DevelopmentSession* session, TaskEventGroup* event_group, AppEventSubscription* subscription, bool* close_requested);

Loading
Loading