JetBrains Space - Issue Grabber Scripts (MIT License)

dotenv

# .env file
SPACE_ORG="your-org.jetbrains.space"
SPACE_DUMPS="/path/for/the/data"
SPACE_TOKEN="<personal-token>"

grab-issue.sh

#!/usr/bin/env bash

set -e
cd "${BASH_SOURCE%/*}" || exit

SPACE_DUMPS="."
SPACE_ORG="org.jetbrains.space"
SPACE_TOKEN="<personal-token>"
source .env

if [ -z "$1" ]; then
  echo "You need to specify the issue (like PROJ-T-123)"
  exit 1
fi

ISSUE="$1"
PROJECT=${ISSUE%%-*}
FROM="2000-01-01T00%3A00%3A00.000Z"
OUTDIR="$SPACE_DUMPS/$PROJECT/$ISSUE"
COMMENTS_FILE="$OUTDIR/comments.json"
ISSUE_FILE="$OUTDIR/issue.json"
ATTACHMENTS="$OUTDIR/attachments"

mkdir -p "$OUTDIR"

# Store the issue itself
curl -s "https://$SPACE_ORG/api/http/issues?issueId=key:$ISSUE&\$fields=id,assignee(username),attachmentsCount,commentsCount,createdBy(name),creationTime,deletedBy(name),deletedTime,dueDate,number,status(name,resolved,archived),tags(name),title,attachments(details(id,filename,name,deletedIdentity)),customFields,description" \
  -H "Authorization: Bearer $SPACE_TOKEN" \
  -H 'Accept: application/json' | jq . > "$ISSUE_FILE"
echo "Got issue.json for $ISSUE"

MAX_ITERATIONS=50

TEMP_FILE=$(mktemp)

# Initialize an empty array for all messages
echo '[]' > "$TEMP_FILE"

for ((i=0; i<MAX_ITERATIONS; i++)); do
    RESPONSE=$(curl -s "https://$SPACE_ORG/api/http/chats/messages?channel=issue:key:$ISSUE&startFromDate=$FROM&sorting=FromOldestToNewest&batchSize=50&\$fields=nextStartFromDate,messages(id,created,edited,text,time,author(name),details(className),attachments(details(id,filename,name,unfurl(details(tag(name),strikeThrough)))),reactions(emojiReactions(emoji,meReacted,count)))" \
         -H "Authorization: Bearer $SPACE_TOKEN" \
         -H 'Accept: application/json')

    # Extract messages from the response and add to the temporary file
    echo "$RESPONSE" | jq '.messages' | jq -s 'add' "$TEMP_FILE" - > "$TEMP_FILE.new" && mv "$TEMP_FILE.new" "$TEMP_FILE"

    LENGTH=$(echo "$RESPONSE" | jq '.messages | length')
    echo "Got $LENGTH messages in query $((i+1))"

    if [[ "$LENGTH" != 50 ]]; then
        break
    fi

    if [[ $i -eq $((MAX_ITERATIONS-1)) ]]; then
        echo "NOT EXPECTED"
        rm "$TEMP_FILE"
        exit 1
    fi

    FROM=$(echo "$RESPONSE" | jq -r '.nextStartFromDate.iso')
done

# Wrap the final array in an object
jq '{ messages: . }' "$TEMP_FILE" > "$COMMENTS_FILE"

rm "$TEMP_FILE"

LENGTH=$(jq '.messages | length' < "$COMMENTS_FILE")
echo "All $LENGTH messages have been concatenated into 'comments.json'"

# loading the issue attachments
LIST=$(jq -r 'select(.attachments | length > 0).attachments[].details | select(.className != "UnfurlAttachment") | [ .className, .id, .name+.filename ] | @csv' < "$ISSUE_FILE")

echo "$LIST" | while IFS=',' read -r TYPE ID NAME
do
    TYPE=$(echo $TYPE | tr -d '"')
    ID=$(echo $ID | tr -d '"')
    NAME=$(echo $NAME | tr -d '"')

    # Skip empty lines
    [ -z "$TYPE" ] && continue
    [ "$TYPE" = "DeletedAttachment" ] && continue

    # skip existing attachments
    if [ -f "$ATTACHMENTS/$ID/$NAME" ]; then
      echo "Skipping existing \"$NAME\" ($ID / $TYPE)"
      continue
    fi

    # Print the extracted values
    mkdir -p "$ATTACHMENTS/$ID"
    echo "Downloading \"$NAME\" ($ID / $TYPE)"
    curl -s -L "https://$SPACE_ORG/d/$ID?f=0&download=true" -H "Authorization: Bearer $SPACE_TOKEN" -o "$ATTACHMENTS/$ID/$NAME"
done

# loading the comment attachments
LIST=$(jq -r '.messages[] | select(.attachments | length > 0).attachments[].details | select(.className != "UnfurlAttachment") | [ .className, .id, .name+.filename ] | @csv' < "$COMMENTS_FILE")

echo "$LIST" | while IFS=',' read -r TYPE ID NAME
do
    TYPE=$(echo $TYPE | tr -d '"')
    ID=$(echo $ID | tr -d '"')
    NAME=$(echo $NAME | tr -d '"')

    # Skip empty lines
    [ -z "$TYPE" ] && continue
    [ "$TYPE" = "DeletedAttachment" ] && continue

    # skip existing attachments
    if [ -f "$ATTACHMENTS/$ID/$NAME" ]; then
      echo "Skipping existing \"$NAME\" ($ID / $TYPE)"
      continue
    fi

    # Print the extracted values
    mkdir -p "$ATTACHMENTS/$ID"
    echo "Downloading \"$NAME\" ($ID / $TYPE)"
    curl -s -L "https://$SPACE_ORG/d/$ID?f=0&download=true" -H "Authorization: Bearer $SPACE_TOKEN" -o "$ATTACHMENTS/$ID/$NAME"
done

grab-project-issues.sh

#!/usr/bin/env bash

set -e
cd "${BASH_SOURCE%/*}" || exit

PROJECT=""

SPACE_DUMPS="."
SPACE_ORG="org.jetbrains.space"
SPACE_TOKEN="<personal-token>"
source .env

if [ -n "$1" ]; then
  PROJECT="$1"
else [ -z "$PROJECT" ];
  echo "No project given!"
fi

mkdir -p "$SPACE_DUMPS/$PROJECT/"

echo "Fetching all issues from project $PROJECT"
# fetch a fresh list of all issues in the project (minus the deleted ones)
curl -s "https://$SPACE_ORG/api/http/projects/key:$PROJECT/planning/issues?\$top=1000&sorting=CREATED&descending=false&\$fields=data(id,createdBy(name),creationTime,deletedTime,commentsCount,channel(totalMessages),number,status(name),tags(name),title,attachments(details(filename,id,name)),channel(id,totalMessages),customFields,description,dueDate,assignee(username))" -H "Authorization: Bearer $SPACE_TOKEN" -H 'Accept: application/json' | jq . > "$SPACE_DUMPS/$PROJECT/jb-issues.json"

# now walk the list and get all comments and attachments for them
LIST=$(jq -r '.data[] | [ .number, .status.name ] | @csv' < "$SPACE_DUMPS/$PROJECT/jb-issues.json")

COUNT=1

echo "$LIST" | while IFS=',' read -r NUMBER STATUS
do
  NUMBER=$(echo "$NUMBER" | tr -d '"')
  #echo "AT: $COUNT / $NUMBER"
  STATUS=$(echo "$STATUS" | tr -d '"')
  # we also read all deleted ones
  for ((i = $COUNT; i <= $NUMBER; i++)); do
    ISSUE="$PROJECT-T-$i"
    if [ "$i" = "$NUMBER" ]; then
      ST="$STATUS"
    else
      ST="Deleted"
    fi
    if [ -d "$SPACE_DUMPS/$PROJECT/$ISSUE" ]; then
      [ "$ST" = "Deleted" ] && echo "Skipping $ST issue: $ISSUE" && continue
      #[ "$ST" = "Abgerechnet" ] && echo "Skipping $ST issue: $ISSUE" && continue
      #[ "$ST" = "Angedacht" ] && echo "Skipping $ST issue: $ISSUE" && continue
      #[ "$ST" = "Erledigt" ] && echo "Skipping $ST issue: $ISSUE" && continue
      #[ "$ST" = "Zurückgestellt" ] && echo "Skipping $ST issue: $ISSUE" && continue
      #[ "$ST" = "Planung" ] && echo "Skipping $STATUS issue: $PROJECT-T-$NUMBER" && continue
    fi
    echo "Grabbing $ST issue: $ISSUE"
    ./grab-issue.sh "$ISSUE"
  done
  COUNT=$((NUMBER+1))
done

# grabbing the tags used in the project
curl "https://$SPACE_ORG/api/http/projects/key:$PROJECT/planning/tags?\$fields=data(id,name)" -H "Authorization: Bearer $SPACE_TOKEN" -H "Accept: application/json" | jq . > "$SPACE_DUMPS/$PROJECT/jb-tags.json"

SPACE-ISSUE-GRABBER.md

JetBrains Space - Issues JSON export

BEWARE: This is provided as is and we do not give any support to using it. It may or may not work for you!

These bash scripts are meant to export all the issues of JetBrains Space projects.

They were create on OSX and you need jq and curl installed.

There is one script for exporting a single issue and one to export all of the Issues (up to 1000 is the limit)

You need to edit the files or create a .env file with your organisation and a personal token.

• Check if it works for you by using: ./grab-issue.sh PROJECT-T-1
• You can download all (up to 1000) issues using: grab-project-issues.sh PROJECT.

Licence

MIT License

Copyright (c) 2024 Hans Raaf / METATEXX GmbH

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

How can i download more than 1000 (say around 5000) issues?

I was lucky that our single projects were all "just below" that limit. I guess you can add something like $skip=1000 in line 23. Best to make this an argument to the script and manually call it multiple times. It should then skip the first 1000 and grab the next 1000 issues and as it download them individual it should add it to the existing ones.

I was lucky that our single projects were all "just below" that limit. I guess you can add something like $skip=1000 in line 23. Best to make this an argument to the script and manually call it multiple times. It should then skip the first 1000 and grab the next 1000 issues and as it download them individual it should add it to the existing ones.

thank you! also, im not sure why, but in some cases, the log says "Skipping Deleted issue: .." even when that issue is not Deleted, and the folder for it is created and exported normally. When the issue really is Deleted, its treated the same way, ie, its not being skipped.

Ah yes. You need to update the count in line 29 too. It starts at 1 and is was needed for us to have the right sequence. Space will not return deleted issues. So when they are going 1,2,4,5 we count 1,2,3 (deleted), 4,5. Maybe define that higher up in the script and use the value -1 for the "$skip".

oh I see! the way the script is set up now seems like a good way to actually get the deleted issues, not otherwise accessible though Space. since it created folders for every single issue number without skipping any, and put the deleted issues in there too. i guess the reason i was seeing "Skipping Deleted issue" logs for non-deleted items was because of that count offset.

Yes. I wrote some modifications for the Gitea and extended their API and the API SDK with a "time travel" feature. This let me insert all the issues, attachments, comment into Gitea based repositories. Converting status to labels and custom fields to either labels or front matter style data in the head of the issue text. I also had all our git repositories rewritten so that the commit messages refer to the new Gitea based repositories. This came out very good, but was a lot of work. Maybe the worst thing was that I converted data before we made the last payment cycle. As I tracked all the time and invoice data in the custom fields I had to write a "diff" tool later on, to add the missing data into Gitea. I just finished that last weekend, so that we can now calculate the new invoices and know what the clients already paid for. I am so mad at JetBrains for their actions.