Build containers with GitLab CI without root nor daemons by using buildkit

.gitlab-ci.yml

build-container:
  stage: build
  image:
    name: moby/buildkit:rootless
    entrypoint: [ "sh", "-c" ]
  variables:
    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
  before_script:
  - |
    mkdir ~/.docker
    auth=$(echo -n "$CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD" | base64)
    echo "{\"auths\": {\"$CI_REGISTRY\": {\"auth\": \"$auth\"}}}" > ~/.docker/config.json
  script:
  - |
    buildctl-daemonless.sh build \
        --frontend=dockerfile.v0 \
        --local context=. \
        --local dockerfile=. \
        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG \
        --export-cache type=inline \
        --output type=image,name=$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG,push=true

The runner still needs to be privileged, otherwise I get

could not connect to unix:///run/user/1000/buildkit/buildkitd.sock after 42 trials
 ========== log ==========
[rootlesskit:child ] error: failed to share mount point: /: permission denied
[rootlesskit:parent] error: child exited: exit status 1
sh: can't kill pid 52: No such process

Did anyone manage to run this without privileged?