Svetoviz

Activation Functions

Non-linear transformations are the "engines" of neural networks, allowing them to learn complex decision boundaries.

activation_demo.py

                import svetoviz_webgpu as sv

def demo_activation():
    input_tensor = torch.from_numpy(img_np)
                    .view(1, 1, height, width)
                    .repeat(1, 2, 1, 1)

    activations = nn.ModuleDict({
        "ReLU": nn.ReLU(),
        "LeakyReLU": nn.LeakyReLU(negative_slope=0.02),
        "PReLU": nn.PReLU(num_parameters=2),
        "GELU": nn.GELU(),
        "SiLU": nn.SiLU(),
        "Mish": nn.Mish(),
        "Hardswish": nn.Hardswish(),
        "Sigmoid": nn.Sigmoid(),
        "Tanh": nn.Tanh(),
        "Softmax": nn.Softmax(dim=1),
        "LogSoftmax": nn.LogSoftmax(dim=-1),
        "ELU": nn.ELU(),
        "SELU": nn.SELU(),
        "Softplus": nn.Softplus(),
        "Threshold": nn.Threshold(threshold=0.1, value=-2.0),
        "Softshrink": nn.Softshrink(lambd=0.5),
        "GLU": nn.GLU(dim=1),
        "Hardshrink": nn.Hardshrink(lambd=0.5),
        "Hardsigmoid": nn.Hardsigmoid(),
        "Hardtanh": nn.Hardtanh(min_val=-1.0, max_val=1.0),
        "LogSigmoid": nn.LogSigmoid(),
        "ReLU6": nn.ReLU6(),
        "RReLU": nn.RReLU(),
        "CELU": nn.CELU(),
        "Softsign": nn.Softsign(),
        "Tanhshrink": nn.Tanhshrink(),
        "Softmin": nn.Softmin(dim=1),
        "Softmax2d": nn.Softmax2d()
    })

    def terminal_callback(buffer, message, images, files):
        for name, module in activations.items():
            output = module(input_tensor)
            buffer.send_system_message(f"Applied {name}: Output shape {list(output.shape)}")

    sv.pytorch_web(module=activations, terminal_callback=terminal_callback)
                    