stable-diffusion textual inversion training autograd anomaly

autograd.txt

/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/autograd/__init__.py:173: UserWarning: Error detected in NativeLayerNormBackward0. Traceback of forward call that caused the error:
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/Users/birch/.vscode/extensions/ms-python.python-2022.14.0/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/__main__.py", line 39, in <module>
    cli.main()
  File "/Users/birch/.vscode/extensions/ms-python.python-2022.14.0/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 430, in main
    run()
  File "/Users/birch/.vscode/extensions/ms-python.python-2022.14.0/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 284, in run_file
    runpy.run_path(target, run_name="__main__")
  File "/Users/birch/.vscode/extensions/ms-python.python-2022.14.0/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 321, in run_path
    return _run_module_code(code, init_globals, run_name,
  File "/Users/birch/.vscode/extensions/ms-python.python-2022.14.0/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 135, in _run_module_code
    _run_code(code, mod_globals, init_globals,
  File "/Users/birch/.vscode/extensions/ms-python.python-2022.14.0/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 124, in _run_code
    exec(code, run_globals)
  File "/Users/birch/git/stable-diffusion/main.py", line 812, in <module>
    trainer.fit(model, data)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 696, in fit
    self._call_and_handle_interrupt(
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _call_and_handle_interrupt
    return trainer_fn(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 735, in _fit_impl
    results = self._run(model, ckpt_path=self.ckpt_path)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1166, in _run
    results = self._run_stage()
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1252, in _run_stage
    return self._run_train()
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1283, in _run_train
    self.fit_loop.run()
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
    self.advance(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 271, in advance
    self._outputs = self.epoch_loop.run(self._data_fetcher)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
    self.advance(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 203, in advance
    batch_output = self.batch_loop.run(kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
    self.advance(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 87, in advance
    outputs = self.optimizer_loop.run(optimizers, kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
    self.advance(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 201, in advance
    result = self._run_optimization(kwargs, self._optimizers[self.optim_progress.optimizer_position])
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 248, in _run_optimization
    self._optimizer_step(optimizer, opt_idx, kwargs.get("batch_idx", 0), closure)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 358, in _optimizer_step
    self.trainer._call_lightning_module_hook(
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1550, in _call_lightning_module_hook
    output = fn(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1705, in optimizer_step
    optimizer.step(closure=optimizer_closure)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 168, in step
    step_output = self._strategy.optimizer_step(self._optimizer, self._optimizer_idx, closure, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 216, in optimizer_step
    return self.precision_plugin.optimizer_step(model, optimizer, opt_idx, closure, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 153, in optimizer_step
    return optimizer.step(closure=closure, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/optim/optimizer.py", line 113, in wrapper
    return func(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
    return func(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/optim/adamw.py", line 119, in step
    loss = closure()
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 138, in _wrap_closure
    closure_result = closure()
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 146, in __call__
    self._result = self.closure(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 132, in closure
    step_output = self._step_fn()
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 407, in _training_step
    training_step_output = self.trainer._call_strategy_hook("training_step", *kwargs.values())
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1704, in _call_strategy_hook
    output = fn(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/strategies/dp.py", line 134, in training_step
    return self.model(*args, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/parallel/data_parallel.py", line 150, in forward
    return self.module(*inputs, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/overrides/data_parallel.py", line 65, in forward
    output = super().forward(*inputs, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/pytorch_lightning/overrides/base.py", line 79, in forward
    output = self.module.training_step(*inputs, **kwargs)
  File "/Users/birch/git/stable-diffusion/ldm/models/diffusion/ddpm.py", line 346, in training_step
    loss, loss_dict = self.shared_step(batch)
  File "/Users/birch/git/stable-diffusion/ldm/models/diffusion/ddpm.py", line 898, in shared_step
    loss = self(x, c)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/Users/birch/git/stable-diffusion/ldm/models/diffusion/ddpm.py", line 910, in forward
    return self.p_losses(x, c, t, *args, **kwargs)
  File "/Users/birch/git/stable-diffusion/ldm/models/diffusion/ddpm.py", line 1046, in p_losses
    model_output = self.apply_model(x_noisy, t, cond)
  File "/Users/birch/git/stable-diffusion/ldm/models/diffusion/ddpm.py", line 1018, in apply_model
    x_recon = self.model(x_noisy, t, **cond)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/Users/birch/git/stable-diffusion/ldm/models/diffusion/ddpm.py", line 1478, in forward
    out = self.diffusion_model(x, t, context=cc)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/Users/birch/git/stable-diffusion/ldm/modules/diffusionmodules/openaimodel.py", line 737, in forward
    h = module(h, emb, context)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/Users/birch/git/stable-diffusion/ldm/modules/diffusionmodules/openaimodel.py", line 85, in forward
    x = layer(x, context)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/Users/birch/git/stable-diffusion/ldm/modules/attention.py", line 266, in forward
    x = block(x, context=context)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/Users/birch/git/stable-diffusion/ldm/modules/attention.py", line 216, in forward
    return checkpoint(self._forward, (x, context), self.parameters(), self.checkpoint)
  File "/Users/birch/git/stable-diffusion/ldm/modules/diffusionmodules/util.py", line 116, in checkpoint
    return func(*inputs)
  File "/Users/birch/git/stable-diffusion/ldm/modules/attention.py", line 222, in _forward
    x += self.ff(self.norm3(x))
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/modules/normalization.py", line 189, in forward
    return F.layer_norm(
  File "/Users/birch/anaconda3/envs/ldmwaifu-stable/lib/python3.10/site-packages/torch/nn/functional.py", line 2503, in layer_norm
    return torch.layer_norm(input, normalized_shape, weight, bias, eps, torch.backends.cudnn.enabled)
 (Triggered internally at  /Users/runner/work/_temp/anaconda/conda-bld/pytorch_1659484612588/work/torch/csrc/autograd/python_anomaly_mode.cpp:104.)
  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
Summoning checkpoint.