Skip to content

plurigrid/asi / ies/music-topos/.codex/skills/feedforward-learning-local

feedforward-learning-local

Verified

FileKindSize
SKILL.mdskill md2.5 KB
ff_layer.jlother4.4 KB

ff_layer.jl

View on GitHub
"""
Forward-Forward Layer Implementation

Implements Hinton's forward-forward algorithm with local goodness functions
and layer-wise contrastive learning.
"""

module FFLayerModule

export FFLayer, goodness, train_step!, forward

using LinearAlgebra
using Statistics

"""
Forward-Forward Layer
"""
mutable struct FFLayer
    input_dim::Int
    hidden_dim::Int

    # Weights and biases
    W::Matrix{Float64}
    b::Vector{Float64}

    # Goodness threshold
    threshold::Float64

    # Learning rate
    lr::Float64

    # Normalization statistics
    running_mean::Vector{Float64}
    running_var::Vector{Float64}
    momentum::Float64

    function FFLayer(; input_dim::Int, hidden_dim::Int, threshold::Float64=2.0, lr::Float64=0.03)
        W = randn(hidden_dim, input_dim) * sqrt(2.0 / input_dim)
        b = zeros(hidden_dim)
        running_mean = zeros(hidden_dim)
        running_var = ones(hidden_dim)

        new(input_dim, hidden_dim, W, b, threshold, lr, running_mean, running_var, 0.9)
    end
end

"""
Forward pass with ReLU activation
"""
function forward(layer::FFLayer, x::Vector{Float64}; normalize::Bool=true)
    # Linear transform
    z = layer.W * x + layer.b

    # ReLU activation
    h = max.(z, 0.0)

    # Layer normalization (optional)
    if normalize
        h = (h .- layer.running_mean) ./ sqrt.(layer.running_var .+ 1e-8)
    end

    return h
end

function forward(layer::FFLayer, X::Matrix{Float64}; normalize::Bool=true)
    # Batch forward pass
    H = similar(X, layer.hidden_dim, size(X, 2))

    for i in 1:size(X, 2)
        H[:, i] = forward(layer, X[:, i]; normalize=normalize)
    end

    return H
end

"""
Goodness function: sum of squared activities
"""
function goodness(layer::FFLayer, x::Vector{Float64})
    h = forward(layer, x; normalize=false)
    return sum(h .^ 2)
end

"""
Training step: contrastive learning on positive/negative data
"""
function train_step!(layer::FFLayer, x_pos::Vector{Float64}, x_neg::Vector{Float64})
    # Forward pass for positive and negative
    h_pos = forward(layer, x_pos; normalize=false)
    h_neg = forward(layer, x_neg; normalize=false)

    # Compute goodness
    g_pos = sum(h_pos .^ 2)
    g_neg = sum(h_neg .^ 2)

    # Update running statistics
    update_statistics!(layer, h_pos)

    # Gradient of goodness with respect to W
    # Want to increase g_pos and decrease g_neg

    # Positive gradient (increase goodness)
    if g_pos < layer.threshold
        # ∇_W g = 2 h h^T, ∇_W h = diag(h > 0) W, ∇_z h = diag(h > 0)
        # ∇_W g = 2 h (∇_W h) = 2 h x^T where h = σ(Wx + b)
        dW_pos = 2.0 * h_pos * x_pos'
        db_pos = 2.0 * h_pos
    else
        dW_pos = zeros(size(layer.W))
        db_pos = zeros(size(layer.b))
    end

    # Negative gradient (decrease goodness)
    if g_neg > layer.threshold
        dW_neg = -2.0 * h_neg * x_neg'
        db_neg = -2.0 * h_neg
    else
        dW_neg = zeros(size(layer.W))
        db_neg = zeros(size(layer.b))
    end

    # Update weights
    layer.W += layer.lr * (dW_pos + dW_neg)
    layer.b += layer.lr * (db_pos + db_neg)

    return (g_pos=g_pos, g_neg=g_neg)
end

"""
Update running statistics for normalization
"""
function update_statistics!(layer::FFLayer, h::Vector{Float64})
    layer.running_mean = layer.momentum * layer.running_mean + (1 - layer.momentum) * h
    layer.running_var = layer.momentum * layer.running_var + (1 - layer.momentum) * (h .^ 2)
end

"""
Generate negative data by overlaying wrong label
"""
function overlay_wrong_label(x::Vector{Float64}, correct_label::Int, n_classes::Int=10)
    # Assume first n_classes dimensions encode label as one-hot
    x_neg = copy(x)

    # Zero out correct label
    x_neg[correct_label] = 0.0

    # Set random wrong label
    wrong_label = mod(correct_label + rand(1:(n_classes-1)), n_classes)
    x_neg[wrong_label == 0 ? n_classes : wrong_label] = 1.0

    return x_neg
end

"""
Alternative: add noise as negative data
"""
function add_noise_negative(x::Vector{Float64}, noise_level::Float64=0.1)
    return x + noise_level * randn(length(x))
end

"""
Batch training step
"""
function train_step!(layer::FFLayer, X_pos::Matrix{Float64}, X_neg::Matrix{Float64})
    n_samples = size(X_pos, 2)
    total_g_pos = 0.0
    total_g_neg = 0.0

    for i in 1:n_samples
        result = train_step!(layer, X_pos[:, i], X_neg[:, i])
        total_g_pos += result.g_pos
        total_g_neg += result.g_neg
    end

    return (avg_g_pos=total_g_pos/n_samples, avg_g_neg=total_g_neg/n_samples)
end

end  # module
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall