Laravel Sort by Relationship Value

Contact.php

<?php

namespace App;

use App\Helpers\Formatting;
use Illuminate\Database\Eloquent\SoftDeletes;
use Propaganistas\LaravelPhone\PhoneNumber;

class Contact extends Model
{   
    /**
     * Threads
     *
     * Return the threads the contact
     * is a part of.
     *
     * @return \Illuminate\Database\Eloquent\Relations\HasMany
     */
    public function threads() 
    {
        return $this->hasMany(Thread::class);
    }
    
    /**
     * Get Name Attribute
     *
     * Returns the first and last names combined.
     *
     * @return string
     */
    public function getNameAttribute()
    {
        return $this->first_name.' '.$this->last_name;
    }
    
    /**
     * Order By Name
     *
     * Custom query extension
     * that automatically adds
     * the orderBy names.
     *
     * @param $query
     */
    public function scopeOrderByName($query)
    {
        $query->orderBy('last_name')->orderBy('first_name');
    }
    
    /**
     * Filter
     *
     * Creates a function on the query
     * builder called filter.
     *
     * @param       $query
     * @param array $filters
     */
    public function scopeFilter($query, array $filters)
    {
        $query->when($filters['search'] ?? null, function ($query, $search) {
            $query->where(function ($query) use ($search) {
                foreach(explode(' ', $search) as $s) {
                    if(is_numeric($s)) {
                        $s = Formatting::format_phone($s);
                    }
                    $query->where('first_name', 'like', '%' . $s . '%')
                        ->orWhere('last_name', 'like', '%' . $s . '%')
                        ->orWhere('email', 'like', '%' . $s . '%')
                        ->orWhere('primary_phone', 'like', '%' . $s . '%')
                        ->orWhere('secondary_phone', 'like', '%' . $s . '%');
                }
            });
        })->when($filters['status'] ?? null, function ($query, $status) {
            $query->where('status', $status);
        });
    }
}

ContactsController.php

<?php

namespace App\Http\Controllers;

use App\Contact;
use Carbon\Carbon;
use Inertia\Inertia;
use Illuminate\Validation\Rule;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\Request;

class ContactsController extends Controller
{

    /**
     * @return mixed
     */
    public function index ()
    {
        $contacts = Contact::join('threads', 'contacts.id', '=', 'threads.contact_id')
            ->filter(Request::only('search', 'contacts')) //Custom Scope on Contact
            ->orderByDesc('threads.unread_count') //order via threads.unread_count
            ->paginate()
            ->transform(function ($c) {
                return [
                    'id' => $c->id,
                    'name' => $c->name,
                    'phone' => $c->phone,
                    'unread_count' => $c->unread_count,
                ];
            });
        return response()->json($contacts);
    }
}

Thread.php

<?php

namespace App;

class Thread extends Model
{
    /**
     * Contact
     *
     * Returns the contact the thread belongs to.
     *
     * @return \Illuminate\Database\Eloquent\Relations\BelongsTo
     */
    public function contact() 
    {
        return $this->belongsTo(Contact::class);
    }
}

This current version only returns the one contact that has a thread. It should return all contacts, but order them by the field, even if the field doesn't exist, if that makes sense.

For instance:

Contacts

id	name	phone	created_at	updated_at
1	Hello	5555555555	{TIME}	{TIME}
2	World	5555555556	{TIME}	{TIME}

Threads

id	contact_id	unread_count	created_at	updated_at
1	1	15	{TIME}	{TIME}

But the query should return BOTH contacts, regardless of if a thread exists for it.