Merge branch 'arajfer/fix-convergence-in-convnets' into 'internal/main'

[ConvNets/EffNetV2][PyT] Fix grad scaling and dropout

See merge request dl/JoC/examples!1390

PyTorch/Classification/ConvNets/image_classification/models/common.py

@@ -289,13 +289,13 @@ class StochasticDepthResidual(nn.Module):
             return torch.add(residual, other=x)
         else:
             with torch.no_grad():
-                F.dropout(
+                mask = F.dropout(
                     self.mask,
                     p=1 - self.survival_prob,
                     training=self.training,
                     inplace=False,
                 )
-            return torch.addcmul(residual, self.mask, x)
+            return torch.addcmul(residual, mask, x)
 
 class Flatten(nn.Module):
     def forward(self, x: torch.Tensor) -> torch.Tensor:

PyTorch/Classification/ConvNets/main.py

@@ -265,12 +265,6 @@ def add_parser_arguments(parser, skip_arch=False):
         default=1,
         help="Static loss scale, positive power of 2 values can improve amp convergence.",
     )
-    parser.add_argument(
-        "--dynamic-loss-scale",
-        action="store_true",
-        help="Use dynamic loss scaling.  If supplied, this argument supersedes "
-        + "--static-loss-scale.",
-    )
     parser.add_argument(
         "--prof", type=int, default=-1, metavar="N", help="Run only N iterations"
     )
@@ -483,7 +477,7 @@ def prepare_for_training(args, model_args, model_arch):
         init_scale=args.static_loss_scale,
         growth_factor=2,
         backoff_factor=0.5,
-        growth_interval=100 if args.dynamic_loss_scale else 1000000000,
+        growth_interval=100,
         enabled=args.amp,
     )