Skip to content

Commit

Permalink
fixed code formatting on modeling.py
Browse files Browse the repository at this point in the history
  • Loading branch information
julian-fong committed Nov 24, 2024
1 parent ef7a7dd commit 1abcdac
Showing 1 changed file with 5 additions and 8 deletions.
13 changes: 5 additions & 8 deletions src/adapters/methods/modeling.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
from torch import nn

from transformers.activations import get_activation
from transformers.utils.import_utils import is_torchvision_available

from ..configuration import AdapterFusionConfig, BnConfig
from ..context import ForwardContext
from transformers.utils.import_utils import is_torchvision_available


class Activation_Function_Class(nn.Module):
Expand Down Expand Up @@ -132,22 +132,19 @@ def __init__(
elif config["init_weights"] == "houlsby":
for layer in self.adapter_down:
if isinstance(layer, nn.Linear) or isinstance(layer, PHMLayer):
nn.init.trunc_normal_(
layer.weight, mean=0, std=1e-2, a=-2 * 1e-2, b=2 * 1e-2
)
nn.init.trunc_normal_(layer.weight, mean=0, std=1e-2, a=-2 * 1e-2, b=2 * 1e-2)
nn.init.zeros_(layer.bias)

nn.init.trunc_normal_(
self.adapter_up.weight, mean=0, std=1e-2, a=-2 * 1e-2, b=2 * 1e-2
)
nn.init.trunc_normal_(self.adapter_up.weight, mean=0, std=1e-2, a=-2 * 1e-2, b=2 * 1e-2)
nn.init.zeros_(self.adapter_up.bias)
else:
raise ValueError("Unknown init_weights type: {}".format(config["init_weights"]))

if config["stochastic_depth"] > 0.0:
if is_torchvision_available():
from torchvision.ops.stochastic_depth import StochasticDepth
self.DropPath = StochasticDepth(p=config["stochastic_depth"], mode = "row")

self.DropPath = StochasticDepth(p=config["stochastic_depth"], mode="row")
else:
raise ImportError("stochastic_depth requires the package torchvision, but it is not installed")

Expand Down

0 comments on commit 1abcdac

Please sign in to comment.