Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
name: CodeStyle Check
name: "CI: Static Checks"

on:
push:
branches: [main]
pull_request:
branches: [main]

jobs:
code-style-check:
Expand Down
37 changes: 37 additions & 0 deletions .github/workflows/readme-translation-sync.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: README Translation Sync

on:
workflow_dispatch:
push:
branches: [main]
paths:
- README.md

permissions:
contents: write
pull-requests: write
models: read

jobs:
translate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: zrr-lab/readme-translation-sync@main
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
target-language: English
target-file: README.en-US.md
- uses: peter-evans/create-pull-request@v7
with:
commit-message: "📝 docs: sync README.en-US.md translation"
title: "📝 docs: sync README.en-US.md translation"
body: |
This PR automatically syncs the English translation with the latest changes in `README.md`.

Generated using GitHub Models.
branch: readme-translation-sync
delete-branch: true
labels: documentation
25 changes: 19 additions & 6 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,19 +1,32 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.6.0
rev: v6.0.0
hooks:
- id: check-added-large-files
- id: check-case-conflict
- id: check-executables-have-shebangs
- id: check-yaml
- id: check-json
- id: check-toml
- id: check-merge-conflict
- id: debug-statements
- id: destroyed-symlinks
- id: end-of-file-fixer
- id: mixed-line-ending
- id: trailing-whitespace

- repo: https://github.com/tombi-toml/tombi-pre-commit
rev: v0.7.0
hooks:
- id: tombi-format
- id: tombi-lint

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.7.2
rev: v0.14.6
hooks:
- id: ruff-check
args: [--fix]
- id: ruff-format
- id: ruff
args: [--fix, --unsafe-fixes, --exit-non-zero-on-fix]

- repo: https://github.com/crate-ci/typos
rev: v1.33.1
hooks:
- id: typos
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# 多模态情绪识别 (Multimodal Emotion Recognition)
中文 | [English](README.en-US.md)

> 基于深度学习的多模态情绪识别方法研究套件,支持文本、音频、视频三种模态的情绪分类与情感分析任务。

Expand Down
3 changes: 3 additions & 0 deletions _typos.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[default.extend-words]
ACII = "ACII"
NCE = "NCE"
2 changes: 0 additions & 2 deletions configs/encoders/A+V.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@


[model.encoder.A]
feature_size = 768
model = "facebook/data2vec-audio-base-960h"
Expand Down
2 changes: 0 additions & 2 deletions configs/encoders/A.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@


[model.encoder.A]
feature_size = 768
model = "facebook/data2vec-audio-base-960h"
1 change: 1 addition & 0 deletions configs/fusion/DF-0.1.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[model.fusion]
base = "DisentanglementFusion"
args = ["feature_sizes_dict", "256", "128"]

[model.fusion.kwargs]
alpha = "0.1"
1 change: 1 addition & 0 deletions configs/fusion/DF-1.0.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[model.fusion]
base = "DisentanglementFusion"
args = ["feature_sizes_dict", "256", "128"]

[model.fusion.kwargs]
alpha = "1"
1 change: 1 addition & 0 deletions configs/fusion/DF-2.0.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[model.fusion]
base = "DisentanglementFusion"
args = ["feature_sizes_dict", "256", "128"]

[model.fusion.kwargs]
alpha = "2"
1 change: 1 addition & 0 deletions configs/fusion/deep-vallina.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[model.fusion]
base = "VallinaFusionLayer"
args = ["feature_sizes_dict", "128"]

[model.fusion.kwargs]
depth = "5"
1 change: 1 addition & 0 deletions configs/fusion/shared-iemocap.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[model.fusion]
base = "SharedFeatureFusion"
args = ["feature_sizes_dict", "128"]

[model.fusion.kwargs]
alpha = "2"
1 change: 1 addition & 0 deletions configs/fusion/shared-meld.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[model.fusion]
base = "SharedFeatureFusion"
args = ["feature_sizes_dict", "128"]

[model.fusion.kwargs]
alpha = "0.1"
19 changes: 10 additions & 9 deletions experiments/baseline/multimodal.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,18 @@ seed = ["114"]
[tasks.multimodal]
command = "uv run emotion-recognize train"
args = [
"configs/dataset/{dataset}.toml",
"configs/encoders/{encoder}.toml",
"configs/fusion/{fusion}.toml",
"configs/losses/classification/weight.toml",
"--seed {seed}",
"configs/dataset/{dataset}.toml",
"configs/encoders/{encoder}.toml",
"configs/fusion/{fusion}.toml",
"configs/losses/classification/weight.toml",
"--seed {seed}",
]

[tasks.multimodal.matrix]
encoder = [
"T+A",
"T+V",
"A+V",
"T+A+V",
"T+A",
"T+V",
"A+V",
"T+A+V",
]
fusion = ["configs/fusion/vallina.toml"]
15 changes: 8 additions & 7 deletions experiments/baseline/unimodal.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@ classification = ["focal"]
[tasks.unimodal]
command = "uv run emotion-recognize train"
args = [
"configs/dataset/{dataset}.toml",
"configs/encoders/{encoder}.toml",
"configs/losses/classification/{classification}.toml",
"--seed {seed}",
"configs/dataset/{dataset}.toml",
"configs/encoders/{encoder}.toml",
"configs/losses/classification/{classification}.toml",
"--seed {seed}",
]

[tasks.unimodal.matrix]
encoder = [
"T",
"A",
"V",
"T",
"A",
"V",
]
110 changes: 58 additions & 52 deletions experiments/makd-ablation.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ loss = ["focal", "weight"]
[tasks.unimodal]
command = "uv run emotion-recognize train"
args = [
"configs/dataset/{dataset}.toml",
"configs/encoders/T.toml",
"configs/losses/classification/{loss}.toml",
"--seed {seed}",
"configs/dataset/{dataset}.toml",
"configs/encoders/T.toml",
"configs/losses/classification/{loss}.toml",
"--seed {seed}",
]

[tasks.unimodal.matrix]
Expand All @@ -20,97 +20,103 @@ seed = ["0", "114"]
[tasks.meld-losses]
command = "uv run emotion-recognize train"
args = [
"configs/dataset/MELD--E.toml",
"configs/encoders/T+A+V.toml",
"configs/fusion/DF-1.0.toml",
"configs/fusion/kwargs/attn.toml",
"configs/losses/classification/{loss}.toml",
"{extra}",
"--seed 114",
"configs/dataset/MELD--E.toml",
"configs/encoders/T+A+V.toml",
"configs/fusion/DF-1.0.toml",
"configs/fusion/kwargs/attn.toml",
"configs/losses/classification/{loss}.toml",
"{extra}",
"--seed 114",
]
deps = ["unimodal"]

[tasks.meld-losses.matrix]
extra = [
"configs/losses/frl-meld.toml",
"--teacher-checkpoint checkpoints/training/MELD--E/trainable--2--{loss}/1xE--T/ce200638--114",
"configs/losses/frl-meld.toml",
"--teacher-checkpoint checkpoints/training/MELD--E/trainable--2--{loss}/1xE--T/ce200638--114",
]

[tasks.meld-model]
command = "uv run emotion-recognize train"
args = [
"configs/dataset/MELD--E.toml",
"configs/encoders/T+A+V.toml",
"configs/fusion/{fusion}.toml",
"configs/losses/classification/{loss}.toml",
"configs/losses/frl-meld.toml",
"--teacher-checkpoint checkpoints/training/MELD--E/trainable--2--{loss}/1xE--T/ce200638--114",
"--seed 114",
"configs/dataset/MELD--E.toml",
"configs/encoders/T+A+V.toml",
"configs/fusion/{fusion}.toml",
"configs/losses/classification/{loss}.toml",
"configs/losses/frl-meld.toml",
"--teacher-checkpoint checkpoints/training/MELD--E/trainable--2--{loss}/1xE--T/ce200638--114",
"--seed 114",
]
deps = ["unimodal"]

[tasks.meld-model.matrix]
fusion = ["vallina", "shared-meld", "private", "DF-1.0"]

[tasks.meld-modality]
command = "uv run emotion-recognize train"
args = [
"configs/dataset/MELD--E.toml",
"configs/encoders/{encoder}.toml",
"configs/fusion/DF-1.0.toml",
"configs/fusion/kwargs/attn.toml",
"configs/losses/classification/{loss}.toml",
"configs/losses/frl-meld.toml",
"--teacher-checkpoint checkpoints/training/MELD--E/trainable--2--{loss}/1xE--T/ce200638--114",
"--seed 114",
"configs/dataset/MELD--E.toml",
"configs/encoders/{encoder}.toml",
"configs/fusion/DF-1.0.toml",
"configs/fusion/kwargs/attn.toml",
"configs/losses/classification/{loss}.toml",
"configs/losses/frl-meld.toml",
"--teacher-checkpoint checkpoints/training/MELD--E/trainable--2--{loss}/1xE--T/ce200638--114",
"--seed 114",
]
deps = ["unimodal"]

[tasks.meld-modality.matrix]
encoder = ["T+A", "T+V"]

[tasks.iemocap-losses]
command = "uv run emotion-recognize train"
args = [
"configs/dataset/IEMOCAP--E.toml",
"configs/encoders/T+A+V.toml",
"configs/fusion/DF-2.0.toml",
"configs/fusion/kwargs/attn.toml",
"configs/losses/classification/{loss}.toml",
"{extra}",
"--seed 0",
"configs/dataset/IEMOCAP--E.toml",
"configs/encoders/T+A+V.toml",
"configs/fusion/DF-2.0.toml",
"configs/fusion/kwargs/attn.toml",
"configs/losses/classification/{loss}.toml",
"{extra}",
"--seed 0",
]
deps = ["unimodal"]

[tasks.iemocap-losses.matrix]
extra = [
"configs/losses/frl-iemocap.toml",
"--teacher-checkpoint checkpoints/training/IEMOCAP--E/trainable--4--{loss}/1xE--T/ce200638--114",
"configs/losses/frl-iemocap.toml",
"--teacher-checkpoint checkpoints/training/IEMOCAP--E/trainable--4--{loss}/1xE--T/ce200638--114",
]

[tasks.iemocap-model]
command = "uv run emotion-recognize train"
args = [
"configs/dataset/IEMOCAP--E.toml",
"configs/encoders/T+A+V.toml",
"configs/fusion/{fusion}.toml",
"configs/losses/classification/{loss}.toml",
"configs/losses/frl-iemocap.toml",
"--teacher-checkpoint checkpoints/training/IEMOCAP--E/trainable--4--{loss}/1xE--T/ce200638--114",
"--seed 0",
"configs/dataset/IEMOCAP--E.toml",
"configs/encoders/T+A+V.toml",
"configs/fusion/{fusion}.toml",
"configs/losses/classification/{loss}.toml",
"configs/losses/frl-iemocap.toml",
"--teacher-checkpoint checkpoints/training/IEMOCAP--E/trainable--4--{loss}/1xE--T/ce200638--114",
"--seed 0",
]
deps = ["unimodal"]

[tasks.iemocap-model.matrix]
fusion = ["vallina", "shared-iemocap", "private", "DF-2.0"]

[tasks.iemocap-modality]
command = "uv run emotion-recognize train"
args = [
"configs/dataset/IEMOCAP--E.toml",
"configs/encoders/{encoder}.toml",
"configs/fusion/DF-2.0.toml",
"configs/fusion/kwargs/attn.toml",
"configs/losses/classification/{loss}.toml",
"configs/losses/frl-iemocap.toml",
"--teacher-checkpoint checkpoints/training/IEMOCAP--E/trainable--4--{loss}/1xE--T/ce200638--114",
"--seed 0",
"configs/dataset/IEMOCAP--E.toml",
"configs/encoders/{encoder}.toml",
"configs/fusion/DF-2.0.toml",
"configs/fusion/kwargs/attn.toml",
"configs/losses/classification/{loss}.toml",
"configs/losses/frl-iemocap.toml",
"--teacher-checkpoint checkpoints/training/IEMOCAP--E/trainable--4--{loss}/1xE--T/ce200638--114",
"--seed 0",
]
deps = ["unimodal"]

[tasks.iemocap-modality.matrix]
encoder = ["T+A", "T+V"]
Loading