Java snippet that does panning around a JScrollPane by add this as the MouseListener and MouseMotionListener to the viewport of the scroll pane.

RCPanMouseAdapter.java

//~--- JDK imports ------------------------------------------------------------

import java.awt.Cursor;
import java.awt.Point;
import java.awt.Rectangle;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;

import javax.swing.JComponent;
import javax.swing.JViewport;

/**
 *
 * Handles event while user try to drag on the canvas, which results in a
 * image panning effect (hand scrolling).
 * <p>
 * Event handling starts while user presses on the canvas.
 *
 * @author: Cheow Yeong Chi
 *
 * @version 1.4
 *
 */
public class RCPanMouseAdapter extends MouseAdapter {

    /**
     * Panning pivot point
     */
    private final Point panPoint = new Point();

    /**
     * {@inheritDoc}<p>
     * Set pan starting point to be the point where the mouse currently located, and change the
     * current cursor to <code>HAND_CURSOR</code>.
     */
    @Override
    public void mousePressed(MouseEvent event) {
        JViewport  viewport  = (JViewport) event.getSource();      // viewport from the JScrollPane
        JComponent component = (JComponent) viewport.getView();

        panPoint.setLocation(event.getPoint());
        component.setCursor(new Cursor(Cursor.HAND_CURSOR));
    }

    /**
     * {@inheritDoc}<p>
     * Reset cursor to <code>DEFAULT_CURSOR</code>.
     */
    @Override
    public void mouseReleased(MouseEvent event) {
        JViewport  viewport  = (JViewport) event.getSource();
        JComponent component = (JComponent) viewport.getView();

        component.setCursor(new Cursor(Cursor.DEFAULT_CURSOR));
    }

    /**
     * {@inheritDoc}<p>
     * Get the current viewport of the canvas and move the viewable area respective to the
     * event cursor location.
     */
    @Override
    public void mouseDragged(MouseEvent event) {
        JViewport  viewport     = (JViewport) event.getSource();
        JComponent component    = (JComponent) viewport.getView();
        Point      currentPoint = event.getPoint();
        Point      viewPoint    = viewport.getViewPosition();

        viewPoint.translate(panPoint.x - currentPoint.x, panPoint.y - currentPoint.y);
        component.scrollRectToVisible(new Rectangle(viewPoint, viewport.getSize()));
        panPoint.setLocation(currentPoint);
    }
}

I took the general idea from what you had, and put this component together that seems to work. Presently its only set up to pan on a right click drag, but this could be easily changed

import java.awt.Component;
import java.awt.Point;
import java.awt.Rectangle;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;
import java.awt.event.MouseWheelEvent;

import javax.swing.JComponent;
import javax.swing.JPanel;
import javax.swing.JScrollBar;
import javax.swing.JScrollPane;
import javax.swing.JViewport;
import javax.swing.ScrollPaneConstants;
import javax.swing.SwingUtilities;

public class PannableScrollPane extends JPanel implements ScrollPaneConstants{
    private static final long serialVersionUID = 1L;
    // TODO: This _scrollPane should be a lot more generic
    protected JScrollPane _scrollPane = null;
    protected JPanel _glassPane = null;
    private boolean _isEditing = false;

    public class GlassPanePaneMouseAdapter extends MouseAdapter{
        protected Component _workingComponent = null;

        @Override
        public void mouseReleased(MouseEvent e) {
            _workingComponent = null;
            dispatchEvent(e);
        }

        @Override
        public void mousePressed(MouseEvent e) {
            _workingComponent = getComponent(e);
            dispatchEvent(e);
        }

        @Override
        public void mouseExited(MouseEvent e) {
            dispatchEvent(e);
        }

        @Override
        public void mouseEntered(MouseEvent e) {
            dispatchEvent(e);
        }

        @Override
        public void mouseClicked(MouseEvent e) {
            dispatchEvent(e);
        }

        @Override
        public void mouseDragged(MouseEvent e) {
            dispatchEvent(e);
        }

        @Override
        public void mouseMoved(MouseEvent e) {
            dispatchEvent(e);
        }

        @Override
        public void mouseWheelMoved(MouseWheelEvent e) {
            _scrollPane.dispatchEvent(e);
        }

        protected Component getComponent(MouseEvent e){
            Component component = _scrollPane.getComponentAt(e.getPoint());

            while(component != null && component instanceof JViewport){
                component = component.getComponentAt(e.getPoint());
            }

            return component;
        }

        public void dispatchEvent(MouseEvent e){
            if(e != null){
                Component component = _workingComponent;
                if(component == null){
                    component = getComponent(e);
                }

                if(component != null){
                    Point componentPoint = SwingUtilities.convertPoint(_glassPane, e.getPoint(), component);
                    component.dispatchEvent(new MouseEvent(component, e.getID(), e.getWhen(),
                            e.getModifiers(), componentPoint.x, componentPoint.y,
                            e.getClickCount(), e.isPopupTrigger()));
                }
            }
        }
    }

    public class PannableScrollPaneMouseAdapter extends GlassPanePaneMouseAdapter{
        // Number of pixels we have to move before we are in pan mode
        public static final int PAN_MODE_THRESHOLD = 5; 
        protected MouseEvent _startEvent = null;
        protected Point _startPoint = null;
        protected boolean _dragModeEnabled = false;

        @Override
        public void mouseReleased(MouseEvent e) {
            if(isEditing() == true || SwingUtilities.isRightMouseButton(e) == false){
                dispatchEvent(e);
            } else {
                if(_dragModeEnabled == false){
                    dispatchEvent(_startEvent);
                    dispatchEvent(e);
                }
            }

            _workingComponent = null;
            _startEvent = null;
            _startPoint = null;
            _dragModeEnabled = false;
        }

        @Override
        public void mousePressed(MouseEvent e) {
            _workingComponent = getComponent(e);
            _dragModeEnabled = false;
            _startEvent = null;
            _startPoint = null;

            if(isEditing() == true || SwingUtilities.isRightMouseButton(e) == false){
                dispatchEvent(e);
            } else {
                _startEvent = e;
                _startPoint = e.getPoint();
            }
        }

        @Override
        public void mouseDragged(MouseEvent e) {
            if(isEditing() == true || SwingUtilities.isRightMouseButton(e) == false){
                dispatchEvent(e);
            } else {
                // If we are in drag mode, scroll the ScrollPane
                if(_dragModeEnabled == true){
                    JViewport  viewport  = _scrollPane.getViewport();
                    JComponent component = (JComponent) viewport.getView();
                    Point currentPoint   = e.getPoint();
                    Point viewPoint      = _scrollPane.getViewport().getViewPosition();

                    viewPoint.translate(_startPoint.x - currentPoint.x, _startPoint.y - currentPoint.y);
                    component.scrollRectToVisible(new Rectangle(viewPoint, _scrollPane.getViewport().getSize()));
                    _startPoint.setLocation(currentPoint);
                } else {
                    // If we are not in drag mode, dispatch the event
                    if(_startEvent == null){
                        dispatchEvent(e);
                    } 
                    else if(Point.distance(e.getPoint().getX(), e.getPoint().getY(), _startEvent.getPoint().getX(), _startEvent.getPoint().getY()) > PAN_MODE_THRESHOLD){
                        // Initiate Drag Mode
                        _dragModeEnabled = true;
                    } 
                    // else do nothing
                }
                // else do nothing
            }
        }
    }


    public PannableScrollPane(){
            this(null);
    }

    public PannableScrollPane(Component view){
        this(view, VERTICAL_SCROLLBAR_AS_NEEDED, HORIZONTAL_SCROLLBAR_AS_NEEDED);
    }

    public PannableScrollPane(int vsbPolicy, int hsbPolicy){
        this(null, VERTICAL_SCROLLBAR_AS_NEEDED, HORIZONTAL_SCROLLBAR_AS_NEEDED);
    }

    public PannableScrollPane(final Component view, int vsbPolicy, int hsbPolicy){
        super(null);
        _scrollPane = new JScrollPane(view, vsbPolicy, hsbPolicy);
        _glassPane  = new JPanel();
        _glassPane.setOpaque(false);

        PannableScrollPaneMouseAdapter mouseAdapter = new PannableScrollPaneMouseAdapter();
        _glassPane.addMouseListener(mouseAdapter);
        _glassPane.addMouseMotionListener(mouseAdapter);
        _glassPane.addMouseWheelListener(mouseAdapter);

        this.add(_glassPane);
        this.add(_scrollPane);
    }

    public void setViewportView(Component view){
        _scrollPane.setViewportView(view);
    }

    public void setIsEditing(boolean isEditing){
        _isEditing = isEditing;
    }

    public boolean isEditing(){
        return _isEditing;
    }

    public JScrollBar getHorizontalScrollBar(){
        return _scrollPane.getHorizontalScrollBar();
    }

    public JScrollBar getVerticalScrollBar(){
        return _scrollPane.getVerticalScrollBar();
    }

    public void doLayout(){
        super.doLayout();
        _scrollPane.setLocation(this.getLocation());
        _glassPane.setLocation(this.getLocation());
        _scrollPane.setSize(this.getSize());
        _glassPane.setSize(this.getSize());

    }
}

And here is an example of how to use it

import java.awt.BorderLayout;
import java.awt.Dimension;

import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JTable;

public class Main {


    public static void main(String[] args){

        JTable table = new JTable(80, 80);
        for(int j=0; j<80; j++){
            for(int i=0; i<80; i++){
                table.setValueAt("Hello", j, i);
            }
        }
        table.setAutoResizeMode(JTable.AUTO_RESIZE_OFF);
        PannableScrollPane scrollPane = new PannableScrollPane(table);
        JPanel panel = new JPanel(new BorderLayout());
        panel.add(scrollPane, BorderLayout.CENTER);

        JFrame frame = new JFrame("PannableScrollPane Demo");
        frame.setContentPane(panel);
        frame.setLocationByPlatform(true);
        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        frame.setSize(new Dimension(800, 600));
        frame.setVisible(true);
    }
}