Adds WhisperX, reworks UI (still needs some work), theoretically usable
This commit is contained in:
Guillaume Tâche
21
whisper/common/pom.xml
Normal file
21
whisper/common/pom.xml
Normal file
@@ -0,0 +1,21 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.github.gtache.autosubtitle</groupId>
|
||||
<artifactId>autosubtitle-whisper</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>autosubtitle-whisper-common</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.github.gtache.autosubtitle</groupId>
|
||||
<artifactId>autosubtitle-conda</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.github.gtache.autosubtitle.modules.setup.whisper;
|
||||
|
||||
import javax.inject.Qualifier;
|
||||
import java.lang.annotation.Documented;
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
import static java.lang.annotation.RetentionPolicy.RUNTIME;
|
||||
|
||||
@Qualifier
|
||||
@Documented
|
||||
@Retention(RUNTIME)
|
||||
@Target({ElementType.PARAMETER, ElementType.METHOD, ElementType.FIELD})
|
||||
public @interface PythonVersion {
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.github.gtache.autosubtitle.modules.setup.whisper;
|
||||
|
||||
import javax.inject.Qualifier;
|
||||
import java.lang.annotation.Documented;
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
import static java.lang.annotation.RetentionPolicy.RUNTIME;
|
||||
|
||||
@Qualifier
|
||||
@Documented
|
||||
@Retention(RUNTIME)
|
||||
@Target({ElementType.PARAMETER, ElementType.METHOD, ElementType.FIELD})
|
||||
public @interface WhisperBundledRoot {
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.github.gtache.autosubtitle.modules.setup.whisper;
|
||||
|
||||
import com.github.gtache.autosubtitle.impl.OS;
|
||||
import com.github.gtache.autosubtitle.modules.setup.conda.CondaSetupModule;
|
||||
import com.github.gtache.autosubtitle.setup.whisper.WhisperSetupConfiguration;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
@Module(includes = CondaSetupModule.class)
|
||||
public final class WhisperCommonSetupModule {
|
||||
|
||||
private WhisperCommonSetupModule() {
|
||||
|
||||
}
|
||||
|
||||
@Provides
|
||||
static WhisperSetupConfiguration providesWhisperSetupConfiguration(@WhisperBundledRoot final Path root,
|
||||
@WhisperVenvPath final Path venvPath,
|
||||
@PythonVersion final String pythonVersion,
|
||||
final OS os) {
|
||||
return new WhisperSetupConfiguration(root, venvPath, pythonVersion, os);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.github.gtache.autosubtitle.modules.setup.whisper;
|
||||
|
||||
import javax.inject.Qualifier;
|
||||
import java.lang.annotation.Documented;
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
import static java.lang.annotation.RetentionPolicy.RUNTIME;
|
||||
|
||||
@Qualifier
|
||||
@Documented
|
||||
@Retention(RUNTIME)
|
||||
@Target({ElementType.PARAMETER, ElementType.METHOD, ElementType.FIELD})
|
||||
public @interface WhisperVenvPath {
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package com.github.gtache.autosubtitle.setup.whisper;
|
||||
|
||||
import com.github.gtache.autosubtitle.impl.OS;
|
||||
import com.github.gtache.autosubtitle.setup.SetupAction;
|
||||
import com.github.gtache.autosubtitle.setup.SetupException;
|
||||
import com.github.gtache.autosubtitle.setup.SetupManager;
|
||||
import com.github.gtache.autosubtitle.setup.SetupStatus;
|
||||
import com.github.gtache.autosubtitle.setup.conda.CondaSetupManager;
|
||||
import com.github.gtache.autosubtitle.setup.impl.AbstractSetupManager;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
//TODO add gpg/signature check
|
||||
|
||||
/**
|
||||
* Base {@link SetupManager} for Whisper
|
||||
*/
|
||||
public abstract class AbstractWhisperSetupManager extends AbstractSetupManager {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(AbstractWhisperSetupManager.class);
|
||||
private static final String CONDA_ENV = "conda-env";
|
||||
private final CondaSetupManager condaSetupManager;
|
||||
private final WhisperSetupConfiguration configuration;
|
||||
|
||||
protected AbstractWhisperSetupManager(final CondaSetupManager condaSetupManager, final WhisperSetupConfiguration configuration) {
|
||||
this.condaSetupManager = requireNonNull(condaSetupManager);
|
||||
this.configuration = requireNonNull(configuration);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected SetupStatus getStatus() throws SetupException {
|
||||
if (isWhisperInstalled()) {
|
||||
return SetupStatus.BUNDLE_INSTALLED;
|
||||
} else {
|
||||
return SetupStatus.NOT_INSTALLED;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void install() throws SetupException {
|
||||
logger.info("Checking and installing conda");
|
||||
checkInstallConda();
|
||||
logger.info("Checking and creating venv");
|
||||
checkInstallVenv();
|
||||
logger.info("Checking and installing whisper");
|
||||
checkInstallWhisper();
|
||||
logger.info("Install finished");
|
||||
}
|
||||
|
||||
private void checkInstallConda() throws SetupException {
|
||||
sendStartEvent(SetupAction.CHECK, condaSetupManager.name(), 0);
|
||||
if (condaSetupManager.isInstalled()) {
|
||||
sendEndEvent(SetupAction.CHECK, condaSetupManager.name(), 0.33);
|
||||
} else {
|
||||
sendEndEvent(SetupAction.CHECK, condaSetupManager.name(), 0);
|
||||
sendStartEvent(SetupAction.INSTALL, condaSetupManager.name(), -1);
|
||||
condaSetupManager.install();
|
||||
sendEndEvent(SetupAction.INSTALL, condaSetupManager.name(), 0.33);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkInstallVenv() throws SetupException {
|
||||
sendStartEvent(SetupAction.CHECK, CONDA_ENV, 0.33);
|
||||
if (condaSetupManager.venvExists(configuration.venvPath())) {
|
||||
sendEndEvent(SetupAction.CHECK, CONDA_ENV, 0.66);
|
||||
} else {
|
||||
sendEndEvent(SetupAction.CHECK, CONDA_ENV, -1);
|
||||
sendStartEvent(SetupAction.INSTALL, CONDA_ENV, -1);
|
||||
condaSetupManager.createVenv(configuration.venvPath(), configuration.pythonVersion());
|
||||
sendEndEvent(SetupAction.INSTALL, CONDA_ENV, 0.66);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkInstallWhisper() throws SetupException {
|
||||
sendStartEvent(SetupAction.CHECK, name(), 0.66);
|
||||
if (isWhisperInstalled()) {
|
||||
sendEndEvent(SetupAction.CHECK, name(), 1);
|
||||
} else {
|
||||
sendEndEvent(SetupAction.CHECK, name(), 0.66);
|
||||
sendStartEvent(SetupAction.INSTALL, name(), -1);
|
||||
installWhisper();
|
||||
sendEndEvent(SetupAction.INSTALL, name(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void uninstall() throws SetupException {
|
||||
deleteFolder(configuration.root());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void update() throws SetupException {
|
||||
sendStartEvent(SetupAction.CHECK, condaSetupManager.name(), 0);
|
||||
if (condaSetupManager.isUpdateAvailable()) {
|
||||
sendEndEvent(SetupAction.CHECK, condaSetupManager.name(), 0);
|
||||
sendStartEvent(SetupAction.UPDATE, condaSetupManager.name(), -1);
|
||||
condaSetupManager.update();
|
||||
sendEndEvent(SetupAction.UPDATE, condaSetupManager.name(), 1);
|
||||
} else {
|
||||
sendEndEvent(SetupAction.CHECK, condaSetupManager.name(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the path to the python executable
|
||||
*/
|
||||
protected Path getPythonPath() {
|
||||
return configuration.venvPath().resolve(configuration.os() == OS.WINDOWS ? "python.exe" : "python");
|
||||
}
|
||||
|
||||
/**
|
||||
* Installs Whisper
|
||||
*
|
||||
* @throws SetupException if an error occurred
|
||||
*/
|
||||
protected abstract void installWhisper() throws SetupException;
|
||||
|
||||
/**
|
||||
* Check if whisper is installed
|
||||
*
|
||||
* @return true if whisper is installed
|
||||
* @throws SetupException
|
||||
*/
|
||||
protected abstract boolean isWhisperInstalled() throws SetupException;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.github.gtache.autosubtitle.setup.whisper;
|
||||
|
||||
import com.github.gtache.autosubtitle.impl.OS;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Configuration for whisper setup
|
||||
*/
|
||||
public record WhisperSetupConfiguration(Path root, Path venvPath, String pythonVersion, OS os) {
|
||||
|
||||
public WhisperSetupConfiguration {
|
||||
Objects.requireNonNull(root);
|
||||
Objects.requireNonNull(venvPath);
|
||||
Objects.requireNonNull(pythonVersion);
|
||||
Objects.requireNonNull(os);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package com.github.gtache.autosubtitle.subtitle.extractor.whisper;
|
||||
|
||||
import com.github.gtache.autosubtitle.Audio;
|
||||
import com.github.gtache.autosubtitle.File;
|
||||
import com.github.gtache.autosubtitle.Language;
|
||||
import com.github.gtache.autosubtitle.Video;
|
||||
import com.github.gtache.autosubtitle.impl.OS;
|
||||
import com.github.gtache.autosubtitle.process.impl.AbstractProcessRunner;
|
||||
import com.github.gtache.autosubtitle.subtitle.Subtitle;
|
||||
import com.github.gtache.autosubtitle.subtitle.SubtitleCollection;
|
||||
import com.github.gtache.autosubtitle.subtitle.converter.ParseException;
|
||||
import com.github.gtache.autosubtitle.subtitle.converter.SubtitleConverter;
|
||||
import com.github.gtache.autosubtitle.subtitle.extractor.ExtractEvent;
|
||||
import com.github.gtache.autosubtitle.subtitle.extractor.ExtractException;
|
||||
import com.github.gtache.autosubtitle.subtitle.extractor.ExtractionModel;
|
||||
import com.github.gtache.autosubtitle.subtitle.extractor.SubtitleExtractor;
|
||||
import com.github.gtache.autosubtitle.subtitle.extractor.SubtitleExtractorListener;
|
||||
import com.github.gtache.autosubtitle.subtitle.extractor.impl.ExtractEventImpl;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
/**
|
||||
* Base implementation of {@link SubtitleExtractor} for Whisper
|
||||
*/
|
||||
public abstract class AbstractWhisperSubtitleExtractor extends AbstractProcessRunner implements SubtitleExtractor {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(AbstractWhisperSubtitleExtractor.class);
|
||||
|
||||
private static final String AUTOSUBTITLE = "autosubtitle";
|
||||
private static final Pattern PROGRESS_PATTERN = Pattern.compile("^Progress:\\s*(?<progress>\\d{1,3}\\.\\d{1,3}).+");
|
||||
private static final Pattern LINE_PROGRESS_PATTERN = Pattern.compile("^\\[\\d{2}:\\d{2}\\.\\d{3} --> (?<minutes>\\d{2}):(?<seconds>\\d{2})\\.(?<millis>\\d{3})].+");
|
||||
private static final Pattern TQDM_PROGRESS_PATTERN = Pattern.compile("^(?<progress>\\d{1,3})%\\|.+");
|
||||
private final Path venvPath;
|
||||
private final SubtitleConverter converter;
|
||||
private final OS os;
|
||||
private final Set<SubtitleExtractorListener> listeners;
|
||||
|
||||
protected AbstractWhisperSubtitleExtractor(final Path venvPath, final Map<String, SubtitleConverter> converters, final OS os) {
|
||||
this.venvPath = requireNonNull(venvPath);
|
||||
this.converter = requireNonNull(converters.get("json"));
|
||||
this.os = requireNonNull(os);
|
||||
this.listeners = new HashSet<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addListener(final SubtitleExtractorListener listener) {
|
||||
listeners.add(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeListener(final SubtitleExtractorListener listener) {
|
||||
listeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeListeners() {
|
||||
listeners.clear();
|
||||
}
|
||||
|
||||
private void notifyListeners(final ExtractEvent event) {
|
||||
listeners.forEach(listener -> listener.listen(event));
|
||||
}
|
||||
|
||||
@Override
|
||||
public SubtitleCollection<Subtitle> extract(final Video video, final Language language, final ExtractionModel model) throws ExtractException {
|
||||
if (video instanceof final File f) {
|
||||
return extract(f.path(), language, model, video.info().duration());
|
||||
} else {
|
||||
try {
|
||||
final var path = Files.createTempFile(AUTOSUBTITLE, video.info().videoFormat());
|
||||
try (final var in = video.getInputStream()) {
|
||||
Files.copy(in, path);
|
||||
final var ret = extract(path, language, model, video.info().duration());
|
||||
Files.deleteIfExists(path);
|
||||
return ret;
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
throw new ExtractException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public SubtitleCollection<Subtitle> extract(final Audio audio, final Language language, final ExtractionModel model) throws ExtractException {
|
||||
if (audio instanceof final File f) {
|
||||
return extract(f.path(), language, model, audio.info().duration());
|
||||
} else {
|
||||
try {
|
||||
final var path = Files.createTempFile(AUTOSUBTITLE, audio.info().audioFormat());
|
||||
try (final var in = audio.getInputStream()) {
|
||||
Files.copy(in, path);
|
||||
final var ret = extract(path, language, model, audio.info().duration());
|
||||
Files.deleteIfExists(path);
|
||||
return ret;
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
throw new ExtractException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private SubtitleCollection<Subtitle> extract(final Path path, final Language language, final ExtractionModel model, final long duration) throws ExtractException {
|
||||
try {
|
||||
final var outputDir = Files.createTempDirectory(AUTOSUBTITLE);
|
||||
final var args = createArgs(path, language, model, outputDir);
|
||||
final var processListener = startListen(args);
|
||||
var oldProgress = -1.0;
|
||||
var line = processListener.readLine();
|
||||
while (line != null) {
|
||||
logger.info("[whisper]: {}", line);
|
||||
final var newProgress = computeProgress(line, oldProgress, duration);
|
||||
notifyListeners(new ExtractEventImpl(line, newProgress));
|
||||
oldProgress = newProgress;
|
||||
line = processListener.readLine();
|
||||
}
|
||||
final var result = processListener.join(Duration.ofHours(1));
|
||||
if (result.exitCode() == 0) {
|
||||
final var filename = path.getFileName().toString();
|
||||
final var subtitleFilename = filename.substring(0, filename.lastIndexOf('.')) + ".json";
|
||||
final var subtitleFile = outputDir.resolve(subtitleFilename);
|
||||
return parseResult(subtitleFile);
|
||||
} else {
|
||||
throw new ExtractException("Error extracting subtitles: " + result.output());
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
throw new ExtractException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private SubtitleCollection<Subtitle> parseResult(final Path subtitleFile) throws ExtractException {
|
||||
try {
|
||||
return converter.parse(subtitleFile);
|
||||
} catch (final ParseException e) {
|
||||
throw new ExtractException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private static double computeProgress(final CharSequence line, final double oldProgress, final long duration) {
|
||||
final var matcher = LINE_PROGRESS_PATTERN.matcher(line);
|
||||
if (matcher.matches()) {
|
||||
final var minutes = Integer.parseInt(matcher.group("minutes"));
|
||||
final var seconds = Integer.parseInt(matcher.group("seconds"));
|
||||
final var millis = Integer.parseInt(matcher.group("millis"));
|
||||
return ((minutes * 60L + seconds) * 1000 + millis) / (double) duration;
|
||||
} else {
|
||||
final var progressMatcher = PROGRESS_PATTERN.matcher(line);
|
||||
if (progressMatcher.matches()) {
|
||||
final var progress = Double.parseDouble(progressMatcher.group("progress"));
|
||||
return progress / 100.0;
|
||||
} else {
|
||||
final var tqdmMatcher = TQDM_PROGRESS_PATTERN.matcher(line);
|
||||
if (tqdmMatcher.matches()) {
|
||||
final var progress = Integer.parseInt(tqdmMatcher.group("progress"));
|
||||
return progress / 100.0;
|
||||
} else {
|
||||
return oldProgress;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the command line arguments for Whisper
|
||||
*
|
||||
* @param path the path to the file
|
||||
* @param language the language
|
||||
* @param model the model
|
||||
* @param outputDir the output directory
|
||||
* @return the list of arguments
|
||||
*/
|
||||
protected abstract List<String> createArgs(final Path path, final Language language, final ExtractionModel model, final Path outputDir);
|
||||
|
||||
/**
|
||||
* @return the path to the python executable
|
||||
*/
|
||||
protected Path getPythonPath() {
|
||||
return venvPath.resolve(os == OS.WINDOWS ? "python.exe" : "python");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.github.gtache.autosubtitle.whisper;
|
||||
|
||||
import com.github.gtache.autosubtitle.subtitle.extractor.ExtractionModel;
|
||||
import com.github.gtache.autosubtitle.subtitle.extractor.ExtractionModelProvider;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Singleton;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Whisper implementation of {@link ExtractionModelProvider}
|
||||
*/
|
||||
@Singleton
|
||||
public class WhisperExtractionModelProvider implements ExtractionModelProvider {
|
||||
|
||||
@Inject
|
||||
WhisperExtractionModelProvider() {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ExtractionModel> getAvailableExtractionModels() {
|
||||
return Arrays.asList(WhisperModels.values());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtractionModel getDefaultExtractionModel() {
|
||||
return WhisperModels.MEDIUM;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtractionModel getExtractionModel(final String name) {
|
||||
return WhisperModels.valueOf(name.toUpperCase());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.github.gtache.autosubtitle.whisper;
|
||||
|
||||
import com.github.gtache.autosubtitle.subtitle.extractor.ExtractionModel;
|
||||
|
||||
/**
|
||||
* Whisper models
|
||||
*/
|
||||
public enum WhisperModels implements ExtractionModel {
|
||||
TINY, BASE, SMALL, MEDIUM, LARGE;
|
||||
|
||||
public boolean hasEnglishSpecific() {
|
||||
return this != LARGE;
|
||||
}
|
||||
}
|
||||
15
whisper/common/src/main/java/module-info.java
Normal file
15
whisper/common/src/main/java/module-info.java
Normal file
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* Common whisper module
|
||||
*/
|
||||
module com.github.gtache.autosubtitle.whisper.common {
|
||||
requires transitive com.github.gtache.autosubtitle.conda;
|
||||
requires transitive java.net.http;
|
||||
requires org.apache.logging.log4j;
|
||||
requires transitive com.google.gson;
|
||||
requires transitive java.compiler; //Don't know why dagger generates @Generated here, need to debug
|
||||
|
||||
exports com.github.gtache.autosubtitle.whisper;
|
||||
exports com.github.gtache.autosubtitle.setup.whisper;
|
||||
exports com.github.gtache.autosubtitle.subtitle.extractor.whisper;
|
||||
exports com.github.gtache.autosubtitle.modules.setup.whisper;
|
||||
}
|
||||
Reference in New Issue
Block a user