Example of Failing Kaniko Cache

build.sh

#!/bin/bash
# note: the first build will succeed. The second build will fail.
gcloud builds submit --config=cloudbuild.yml |tee run1.txt.log
gcloud builds submit --config=cloudbuild.yml |tee run2.txt.log

cloudbuild.yml

# used with kaniko gcr.io/kaniko-project/executor@sha256:78d44ec4e9cb5545d7f85c1924695c89503ded86a59f92c7ae658afa3cff5400
steps:
 - name: gcr.io/kaniko-project/executor
   args:
      - --dockerfile=./Dockerfile
      - --destination=gcr.io/your-project/kaniko_example # note: change 'your-project' to your google cloud 
      - --cache=true
      - --cache-ttl=24h

Dockerfile

FROM google/cloud-sdk:256.0.0-alpine

COPY postgres_monitor /usr/bin
RUN apk add --no-cache \ 
      jq=~1.6 \       
      py-pip=~18.1 \
      postgresql-client=~11.5 && \
    chmod +x /usr/bin/postgres_monitor

postgres_monitor

#!/usr/bin/

echo "this is a test"

run1.txt.log

----------------------------------------------------------------- REMOTE BUILD OUTPUT -----------------------------------------------------------------
starting build "8bcbe578-1458-49e6-8c6b-4a20f46835d0"

FETCHSOURCE
Fetching storage object: gs://my-sandbox_cloudbuild/source/1566055493.0-2f3d6f52325c479eb3783245c7f725d7.tgz#1566055493302257
Copying gs://my-sandbox_cloudbuild/source/1566055493.0-2f3d6f52325c479eb3783245c7f725d7.tgz#1566055493302257...
/ [0 files][    0.0 B/  1.6 KiB]                                                
/ [1 files][  1.6 KiB/  1.6 KiB]                                                
Operation completed over 1 objects/1.6 KiB.                                      
BUILD
Pulling image: gcr.io/kaniko-project/executor
Using default tag: latest
latest: Pulling from kaniko-project/executor
Digest: sha256:78d44ec4e9cb5545d7f85c1924695c89503ded86a59f92c7ae658afa3cff5400
Status: Downloaded newer image for gcr.io/kaniko-project/executor:latest
gcr.io/kaniko-project/executor:latest
INFO[0000] Resolved base name google/cloud-sdk:256.0.0-alpine to google/cloud-sdk:256.0.0-alpine 
INFO[0000] Resolved base name google/cloud-sdk:256.0.0-alpine to google/cloud-sdk:256.0.0-alpine 
INFO[0000] Downloading base image google/cloud-sdk:256.0.0-alpine 
2019/08/17 15:25:08 No matching credentials were found, falling back on anonymous
INFO[0000] Error while retrieving image from cache: getting file info: stat /cache/sha256:46cac9de07e621720947afd83a5d6c4a67fc6de8ce710b89f79b9d3f25e12e06: no such file or directory 
INFO[0000] Downloading base image google/cloud-sdk:256.0.0-alpine 
2019/08/17 15:25:08 No matching credentials were found, falling back on anonymous
INFO[0000] Built cross stage deps: map[]                
INFO[0000] Downloading base image google/cloud-sdk:256.0.0-alpine 
2019/08/17 15:25:08 No matching credentials were found, falling back on anonymous
INFO[0001] Error while retrieving image from cache: getting file info: stat /cache/sha256:46cac9de07e621720947afd83a5d6c4a67fc6de8ce710b89f79b9d3f25e12e06: no such file or directory 
INFO[0001] Downloading base image google/cloud-sdk:256.0.0-alpine 
2019/08/17 15:25:09 No matching credentials were found, falling back on anonymous
INFO[0001] Using files from context: [/workspace/postgres_monitor] 
INFO[0001] Checking for cached layer gcr.io/my-sandbox/kaniko_example2/cache:56bb87507adc68a6df9c3240781a3fb1263e1ce34f784ea71d7953a47ecd59d1... 
INFO[0001] No cached layer found for cmd RUN apk add --no-cache       jq=~1.6       py-pip=~18.1       postgresql-client=~11.5 &&     chmod +x /usr/bin/postgres_monitor 
INFO[0001] Unpacking rootfs as cmd RUN apk add --no-cache       jq=~1.6       py-pip=~18.1       postgresql-client=~11.5 &&     chmod +x /usr/bin/postgres_monitor requires it. 
INFO[0005] Taking snapshot of full filesystem...        
INFO[0014] Using files from context: [/workspace/postgres_monitor] 
INFO[0014] COPY postgres_monitor /usr/bin               
INFO[0014] Taking snapshot of files...                  
INFO[0014] RUN apk add --no-cache       jq=~1.6       py-pip=~18.1       postgresql-client=~11.5 &&     chmod +x /usr/bin/postgres_monitor 
INFO[0014] cmd: /bin/sh                                 
INFO[0014] args: [-c apk add --no-cache       jq=~1.6       py-pip=~18.1       postgresql-client=~11.5 &&     chmod +x /usr/bin/postgres_monitor] 
fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/APKINDEX.tar.gz
fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/community/x86_64/APKINDEX.tar.gz
(1/6) Installing oniguruma (6.9.1-r0)
(2/6) Installing jq (1.6-r0)
(3/6) Installing libpq (11.5-r0)
(4/6) Installing postgresql-client (11.5-r0)
(5/6) Installing py-setuptools (40.6.3-r0)
(6/6) Installing py2-pip (18.1-r0)
Executing busybox-1.29.3-r10.trigger
OK: 101 MiB in 61 packages
INFO[0014] Taking snapshot of full filesystem...        
INFO[0018] Pushing layer gcr.io/my-sandbox/kaniko_example2/cache:56bb87507adc68a6df9c3240781a3fb1263e1ce34f784ea71d7953a47ecd59d1 to cache now 
2019/08/17 15:25:28 pushed blob: sha256:cd1c009437c58710cd67ef4b0d4ea843f635fc02493156701eb3e8594dd19b5a
2019/08/17 15:25:28 pushed blob: sha256:1e4222392e2f2e3860211711e40d0c77d3bd93b962e2de5cb453a1f82c4434c6
2019/08/17 15:25:29 gcr.io/my-sandbox/kaniko_example2/cache:56bb87507adc68a6df9c3240781a3fb1263e1ce34f784ea71d7953a47ecd59d1: digest: sha256:482cff081287b2debc400da9a0a188d2f585406bde2f967e64a907ff823ea4f6 size: 428
2019/08/17 15:25:29 existing blob: sha256:1e4222392e2f2e3860211711e40d0c77d3bd93b962e2de5cb453a1f82c4434c6
2019/08/17 15:25:29 existing blob: sha256:50bde9bd59f3e7d8506dd66b9f7e727d4163f52b9a24cff8c4502c8350c8162d
2019/08/17 15:25:29 existing blob: sha256:8e402f1a9c577ded051c1ef10e9fe4492890459522089959988a4852dee8ab2c
2019/08/17 15:25:30 pushed blob: sha256:644977c4d5f598db10210ce64d0f8a8f17494c24677905ef8f7f8ec9c80a8e06
2019/08/17 15:25:31 pushed blob: sha256:d8e23d69891703ed1c2a580cd46e1c53a42243f37574431d92229550774f7bf8
2019/08/17 15:25:31 gcr.io/my-sandbox/kaniko_example2:latest: digest: sha256:e5bf8eca12d6d6c705c57306fdfc43a89cfb44bc955cf4b8d0d5730c65e6344c size: 917
PUSH
DONE
-------------------------------------------------------------------------------------------------------------------------------------------------------

ID                                    CREATE_TIME                DURATION  SOURCE                                                                                   IMAGES  STATUS
8bcbe578-1458-49e6-8c6b-4a20f46835d0  2019-08-17T15:24:53+00:00  39S       gs://my-sandbox_cloudbuild/source/1566055493.0-2f3d6f52325c479eb3783245c7f725d7.tgz  -       SUCCESS

run2.txt.log

----------------------------------------------------------------- REMOTE BUILD OUTPUT -----------------------------------------------------------------
starting build "4e68eef4-2ca9-4472-adf5-a0e58c48bcc8"

FETCHSOURCE
Fetching storage object: gs://my-sandbox_cloudbuild/source/1566055537.3-5d992c224c6b4b4c9596fd809e53d485.tgz#1566055537863054
Copying gs://my-sandbox_cloudbuild/source/1566055537.3-5d992c224c6b4b4c9596fd809e53d485.tgz#1566055537863054...
/ [0 files][    0.0 B/  2.3 KiB]                                                
/ [1 files][  2.3 KiB/  2.3 KiB]                                                
Operation completed over 1 objects/2.3 KiB.                                      
BUILD
Pulling image: gcr.io/kaniko-project/executor
Using default tag: latest
latest: Pulling from kaniko-project/executor
Digest: sha256:78d44ec4e9cb5545d7f85c1924695c89503ded86a59f92c7ae658afa3cff5400
Status: Downloaded newer image for gcr.io/kaniko-project/executor:latest
gcr.io/kaniko-project/executor:latest
INFO[0000] Resolved base name google/cloud-sdk:256.0.0-alpine to google/cloud-sdk:256.0.0-alpine 
INFO[0000] Resolved base name google/cloud-sdk:256.0.0-alpine to google/cloud-sdk:256.0.0-alpine 
INFO[0000] Downloading base image google/cloud-sdk:256.0.0-alpine 
2019/08/17 15:25:54 No matching credentials were found, falling back on anonymous
INFO[0000] Error while retrieving image from cache: getting file info: stat /cache/sha256:46cac9de07e621720947afd83a5d6c4a67fc6de8ce710b89f79b9d3f25e12e06: no such file or directory 
INFO[0000] Downloading base image google/cloud-sdk:256.0.0-alpine 
2019/08/17 15:25:54 No matching credentials were found, falling back on anonymous
INFO[0001] Built cross stage deps: map[]                
INFO[0001] Downloading base image google/cloud-sdk:256.0.0-alpine 
2019/08/17 15:25:55 No matching credentials were found, falling back on anonymous
INFO[0001] Error while retrieving image from cache: getting file info: stat /cache/sha256:46cac9de07e621720947afd83a5d6c4a67fc6de8ce710b89f79b9d3f25e12e06: no such file or directory 
INFO[0001] Downloading base image google/cloud-sdk:256.0.0-alpine 
2019/08/17 15:25:55 No matching credentials were found, falling back on anonymous
INFO[0001] Using files from context: [/workspace/postgres_monitor] 
INFO[0001] Checking for cached layer gcr.io/my-sandbox/kaniko_example2/cache:56bb87507adc68a6df9c3240781a3fb1263e1ce34f784ea71d7953a47ecd59d1... 
INFO[0001] Using caching version of cmd: RUN apk add --no-cache       jq=~1.6       py-pip=~18.1       postgresql-client=~11.5 &&     chmod +x /usr/bin/postgres_monitor 
INFO[0001] Skipping unpacking as no commands require it. 
INFO[0001] Taking snapshot of full filesystem...        
INFO[0001] Using files from context: [/workspace/postgres_monitor] 
INFO[0001] COPY postgres_monitor /usr/bin               
INFO[0001] Taking snapshot of files...                  
INFO[0001] RUN apk add --no-cache       jq=~1.6       py-pip=~18.1       postgresql-client=~11.5 &&     chmod +x /usr/bin/postgres_monitor 
INFO[0001] Found cached layer, extracting to filesystem 
error building image: error building stage: extracting fs from image: open /usr/bin/psql: not a directory
ERROR
ERROR: build step 0 "gcr.io/kaniko-project/executor" failed: exit status 1
-------------------------------------------------------------------------------------------------------------------------------------------------------

I have many other examples of the same, which I can demonstrate in person.