very early JTT prototype, created from https://github.com/dret/JTT/blob/master/GPX/test.gpx via https://github.com/dret/JTT/blob/master/GPX/GPX2JTT/GPX2JTT.xsl

test.json

{ "JTT" : [
   { "track" : {
     "title" : "Test Activity",
     "desc" : "Simple test activity walking around the block with a heart rate monitor, stopping once, and marking one lap. Recorded with Garmin Fenix 3, uploaded to Garmin Connect, and exported as GPX and TCX. Both exports are crippled because they do not contain all data fields of the activity.",
     "segments" : [
       { "data-fields" : ["latitude", "longitude", "elevation", "temperature", "HR" ] }, 
       [
         [ 47.407614681869745, 8.553115781396627, 451.79998779296875, "2015-11-13T12:57:24.000Z", 28.0, 76],
         [ 47.40762138739228, 8.553108656778932, 451.0, "2015-11-13T12:57:25.000Z", 28.0, 76],
         [ 47.407626835629344, 8.553094072267413, 450.0, "2015-11-13T12:57:26.000Z", 28.0, 76],
         [ 47.407668409869075, 8.552960716187954, 449.0, "2015-11-13T12:57:33.000Z", 28.0, 78],
         [ 47.40770772099495, 8.552867090329528, 448.79998779296875, "2015-11-13T12:57:39.000Z", 28.0, 75],
         [ 47.407753402367234, 8.5526111908257, 448.79998779296875, "2015-11-13T12:57:54.000Z", 27.0, 75],
         [ 47.40781215950847, 8.552352441474795, 448.20001220703125, "2015-11-13T12:58:04.000Z", 27.0, 73],
         [ 47.407849207520485, 8.552198382094502, 448.20001220703125, "2015-11-13T12:58:12.000Z", 27.0, 76],
         [ 47.40790771320462, 8.552007442340255, 448.0, "2015-11-13T12:58:23.000Z", 27.0, 75],
         [ 47.40796320140362, 8.551924461498857, 447.79998779296875, "2015-11-13T12:58:29.000Z", 27.0, 78],
         [ 47.4079969804734, 8.551919935271144, 447.6000061035156, "2015-11-13T12:58:32.000Z", 27.0, 78],
         [ 47.408049115911126, 8.551959414035082, 447.20001220703125, "2015-11-13T12:58:37.000Z", 27.0, 77],
         [ 47.408086750656366, 8.552005849778652, 446.79998779296875, "2015-11-13T12:58:41.000Z", 27.0, 76],
         [ 47.40809513255954, 8.552019177004695, 446.6000061035156, "2015-11-13T12:58:42.000Z", 27.0, 73],
         [ 47.408130671828985, 8.552070977166295, 446.20001220703125, "2015-11-13T12:58:46.000Z", 27.0, 70],
         [ 47.40816361270845, 8.552097212523222, 446.0, "2015-11-13T12:58:49.000Z", 27.0, 71],
         [ 47.40818230435252, 8.552138702943921, 445.6000061035156, "2015-11-13T12:58:51.000Z", 27.0, 71],
         [ 47.40825707092881, 8.552154125645757, 445.3999938964844, "2015-11-13T12:58:58.000Z", 27.0, 70],
         [ 47.40829017944634, 8.55216016061604, 445.3999938964844, "2015-11-13T12:59:01.000Z", 27.0, 71],
         [ 47.40830333903432, 8.552153706550598, 445.20001220703125, "2015-11-13T12:59:03.000Z", 27.0, 71],
         [ 47.408308032900095, 8.552153706550598, 445.20001220703125, "2015-11-13T12:59:04.000Z", 27.0, 70],
         [ 47.40834197960794, 8.552186479791999, 445.3999938964844, "2015-11-13T13:01:07.000Z", 27.0, 85],
         [ 47.40833795629442, 8.552188323810697, 445.3999938964844, "2015-11-13T13:01:08.000Z", 27.0, 84],
         [ 47.40833401679993, 8.552198549732566, 445.3999938964844, "2015-11-13T13:01:09.000Z", 27.0, 84],
         [ 47.40832010284066, 8.552253786474466, 445.3999938964844, "2015-11-13T13:01:12.000Z", 27.0, 84],
         [ 47.40828674286604, 8.552344981580973, 445.0, "2015-11-13T13:01:17.000Z", 27.0, 80],
         [ 47.40825455635786, 8.552448330447078, 444.79998779296875, "2015-11-13T13:01:23.000Z", 27.0, 79],
         [ 47.40825262852013, 8.552489904686809, 444.79998779296875, "2015-11-13T13:01:26.000Z", 27.0, 80],
         [ 47.40821675397456, 8.55262778699398, 444.79998779296875, "2015-11-13T13:01:35.000Z", 27.0, 79],
         [ 47.40820610895753, 8.552680592983961, 444.6000061035156, "2015-11-13T13:01:38.000Z", 27.0, 79],
         [ 47.40816168487072, 8.552955100312829, 444.6000061035156, "2015-11-13T13:01:52.000Z", 27.0, 76],
         [ 47.408138466998935, 8.553202534094453, 444.20001220703125, "2015-11-13T13:02:03.000Z", 27.0, 76],
         [ 47.408125810325146, 8.553239582106471, 444.0, "2015-11-13T13:02:05.000Z", 27.0, 75],
         [ 47.40811625495553, 8.553247042000294, 443.79998779296875, "2015-11-13T13:02:06.000Z", 27.0, 75],
         [ 47.40810167044401, 8.553252154961228, 443.6000061035156, "2015-11-13T13:02:07.000Z", 27.0, 76],
         [ 47.40808733738959, 8.553255340084434, 443.6000061035156, "2015-11-13T13:02:08.000Z", 27.0, 77],
         [ 47.408021707087755, 8.553288113325834, 444.20001220703125, "2015-11-13T13:02:13.000Z", 27.0, 80],
         [ 47.4079795461148, 8.55328886769712, 444.6000061035156, "2015-11-13T13:02:17.000Z", 27.0, 82],
         [ 47.40796043537557, 8.553284592926502, 444.79998779296875, "2015-11-13T13:02:19.000Z", 27.0, 83],
         [ 47.40791064687073, 8.553258189931512, 445.20001220703125, "2015-11-13T13:02:23.000Z", 26.0, 86],
         [ 47.40785448811948, 8.553215442225337, 445.6000061035156, "2015-11-13T13:02:27.000Z", 26.0, 89],
         [ 47.40780486725271, 8.553174622356892, 446.20001220703125, "2015-11-13T13:02:32.000Z", 26.0, 92],
         [ 47.40777527913451, 8.553162636235356, 446.79998779296875, "2015-11-13T13:02:35.000Z", 26.0, 94],
         [ 47.40773663856089, 8.553122906014323, 447.3999938964844, "2015-11-13T13:02:39.000Z", 26.0, 96],
         [ 47.40771467797458, 8.553109411150217, 447.6000061035156, "2015-11-13T13:02:41.000Z", 26.0, 98],
         [ 47.407703530043364, 8.55310220271349, 448.0, "2015-11-13T13:02:42.000Z", 26.0, 98],
         [ 47.407659692689776, 8.553062472492456, 448.79998779296875, "2015-11-13T13:02:46.000Z", 26.0, 101],
         [ 47.40764569491148, 8.553053420037031, 449.0, "2015-11-13T13:02:47.000Z", 26.0, 102],
         [ 47.40763236768544, 8.553045960143209, 449.20001220703125, "2015-11-13T13:02:48.000Z", 26.0, 103]
       ]
     ]
   } }
] }

I like where this is going. I always wanted geojson to handle this, but I do agree that a separate standard may be better.
One comment: the data-fields array is missing timestamp from the list:
["latitude", "longitude", "elevation", "temperature", "HR" ] --> ["latitude", "longitude", "elevation", "timestamp", "temperature", "HR" ]

should it be timestamp or time? I see that time is used in the blog post at https://dret.typepad.com/dretblog/2015/11/gps-data-on-the-web.html

The segments list feels a bit wrong to me. I think it would looks better to have a field data as a sibling to data-fields. I also think that the data presented is not a list of segments but rather a list of points ... or something like that.

http://solveig.oslo.no/anchoring-jtt.json ... is this valid JTT?

hard to tell @tobixen, since there is no format definition to validate against. looking at this 6 years later i definitely wouldn't create this "spreadsheet-like" structure with "heading row" with field names and then all "data rows" following that structure. that seems to invite a lot of brittleness.