Some code which should add a few resolutions to xserver using randr extension with xcb.

randr_add_resolution_range.cpp

#include <iostream>
#include <xcb/xcb.h>
#include <xcb/randr.h>
#include <cstring>

class main {
public:
    main();

    xcb_connection_t* conn{};
    xcb_randr_get_screen_resources_reply_t* res{};

    const char *xcb_errors[19] =
            {
                    "Success",
                    "BadRequest",
                    "BadValue",
                    "BadWindow",
                    "BadPixmap",
                    "BadAtom",
                    "BadCursor",
                    "BadFont",
                    "BadMatch",
                    "BadDrawable",
                    "BadAccess",
                    "BadAlloc",
                    "BadColor",
                    "BadGC",
                    "BadIDChoice",
                    "BadName",
                    "BadLength",
                    "BadImplementation",
                    "Unknown"
            };

    inline void handle_xcb_error(xcb_generic_error_t* err, int line) {
        if (!err)
            return;
        uint (*min)(uint, uint) = [](uint a, uint b) { return a < b ? a : b; };
        uint clamped_error_code = min(err->error_code, (sizeof(xcb_errors) / sizeof(xcb_errors[0])) - 1);

        printf("XCB error: %d (%s), sequence: %d, resource id: %d, major code: %d, minor code: %d on line %d\n",
               int(err->error_code), xcb_errors[clamped_error_code],
               int(err->sequence), int(err->resource_id),
               int(err->major_code), int(err->minor_code), line);
        throw std::runtime_error("xcb_error");
    }

#define handle_xcb_error(err) handle_xcb_error(err, __LINE__)

    [[nodiscard]] inline xcb_window_t xcb_root_window() const {
        return xcb_setup_roots_iterator (xcb_get_setup (conn)).data->root;
    }

    bool refresh() {
        xcb_generic_error_t *err;
        res = xcb_randr_get_screen_resources_reply(conn, xcb_randr_get_screen_resources(conn, xcb_root_window()), &err);
        handle_xcb_error(err);
        return res;
    }

    xcb_randr_mode_t getIdForMode(const char* name) {
        refresh();
        char *mode_names = reinterpret_cast<char*>(xcb_randr_get_screen_resources_names(res));
        auto modes = xcb_randr_get_screen_resources_modes(res);
        for (int i = 0; i < xcb_randr_get_screen_resources_modes_length(res); ++i) {
            auto& mode = modes[i];
            char mode_name[64]{};
            snprintf(mode_name, mode.name_len+1, "%s", mode_names);
            mode_names += mode.name_len;

            if (!strcmp(mode_name, name)) {
                return mode.id;
            }
        }
        return 0;
    }

    void createMode(const char* name, int width, int height) {
        xcb_generic_error_t *err;
        bool is_temporary = strcmp(name, "temporary") == 0;
        if (!is_temporary && getIdForMode(name))
            deleteMode(name);

        if (is_temporary && getIdForMode(name))
            return;

        xcb_randr_mode_info_t mode{};
        mode.width = width;
        mode.height = height;
        mode.name_len = strlen(name);
        xcb_randr_create_mode_reply(conn, xcb_randr_create_mode(conn, xcb_root_window(), mode, mode.name_len, const_cast<char*>(name)), &err);
        handle_xcb_error(err);

        if (!refresh()) {
            std::cout << "Could not refresh screen resources" << std::endl;
            return;
        }
        xcb_randr_mode_t mode_id = getIdForMode(name);
        if (!mode_id) {
            std::cout << "Did not found mode \"" << name << "\"" << std::endl;
            return;
        }
        err = xcb_request_check(conn, xcb_randr_add_output_mode_checked(conn, xcb_randr_get_screen_resources_outputs(res)[0], mode_id));
        handle_xcb_error(err);
    }

    void deleteMode(const char* name) {
        xcb_generic_error_t* err;
        xcb_randr_mode_t mode_id = getIdForMode(name);
        if (mode_id) {
            err = xcb_request_check(conn, xcb_randr_delete_output_mode_checked(conn, xcb_randr_get_screen_resources_outputs(res)[0], mode_id));
            handle_xcb_error(err);
            err = xcb_request_check(conn, xcb_randr_destroy_mode_checked(conn, mode_id));
            handle_xcb_error(err);
            refresh();
        }
    }

    void switchToMode(const char *name) {
        xcb_generic_error_t* err{};
        xcb_randr_mode_t mode_id = XCB_NONE;
        xcb_randr_output_t* outputs{};
        int noutput = 0;
        if (name) {
            mode_id = getIdForMode(name);
            if (mode_id == 0) return;
            outputs = xcb_randr_get_screen_resources_outputs(res);
            noutput = xcb_randr_get_screen_resources_outputs_length(res);
        }

        xcb_randr_set_crtc_config_reply(conn, xcb_randr_set_crtc_config(conn, xcb_randr_get_screen_resources_crtcs(res)[0], XCB_CURRENT_TIME,
                                    res->config_timestamp, 0, 0, mode_id, XCB_RANDR_ROTATION_ROTATE_0,
                                  noutput, outputs), &err);
        handle_xcb_error(err);
    };
};

main::main() {
    xcb_generic_error_t* err{};
    setenv("DISPLAY", ":1", 1);
    conn = xcb_connect(nullptr, nullptr);
    int conn_err = xcb_connection_has_error(conn);
    if (conn_err) {
        std::cout << "Connection has error " << err << std::endl;
        xcb_disconnect(conn);
        throw std::runtime_error("Error connecting to server");
    }

    xcb_grab_server(conn);

    const char* temp_mode_name = "temporary";

    int new_width = 1080;
    int new_height = 2400;

    int width_mm = int(25.4*new_width/90);
    int height_mm = int(25.4*new_height/90);

    refresh();

    createMode(temp_mode_name, new_width, new_height);
    switchToMode(nullptr);
    err = xcb_request_check(conn, xcb_randr_set_screen_size_checked(conn, xcb_root_window(), new_width, new_height, width_mm, height_mm));
    handle_xcb_error(err);

    switchToMode(temp_mode_name);

    refresh();
    auto info = xcb_randr_get_output_info_reply(conn, xcb_randr_get_output_info(conn, xcb_randr_get_screen_resources_outputs(res)[0], res->config_timestamp), &err);
    handle_xcb_error(err);
    auto modes = xcb_randr_get_output_info_modes(info);
    xcb_randr_mode_t temp_id = getIdForMode(temp_mode_name);
    for (int i = xcb_randr_get_output_info_modes_length(info)-1; i >= 0; i--) {
        const auto& mode = modes[i];
        if (mode == temp_id) continue;
        if (i == info->num_preferred) continue;
        err = xcb_request_check(conn, xcb_randr_delete_output_mode_checked(conn, xcb_randr_get_screen_resources_outputs(res)[0], mode));
        handle_xcb_error(err);
        xcb_flush(conn);
        err = xcb_request_check(conn, xcb_randr_destroy_mode_checked(conn, mode));
        xcb_flush(conn);
    }

    for (int i=3; i<=10; i++) {
        int w = new_width*i/10;
        int h = new_height*i/10;
        std::string name = std::to_string(w) + "x" + std::to_string(h) + " termux-x11";
        createMode(name.c_str(), w, h);
    }
    std::string name = std::to_string(new_width) + "x" + std::to_string(new_height) + " termux-x11";
    switchToMode(name.c_str());
    deleteMode(temp_mode_name);

    xcb_ungrab_server(conn);
    xcb_flush(conn);
}

int main() {
    [[maybe_unused]] class main main;
}