Add a Task Manager service and update downloads to use it

Bumps [github.com/sirupsen/logrus](https://github.com/sirupsen/logrus) from 1.9.0 to 1.9.3.
- [Release notes](https://github.com/sirupsen/logrus/releases)
- [Changelog](https://github.com/sirupsen/logrus/blob/master/CHANGELOG.md)
- [Commits](https://github.com/sirupsen/logrus/compare/v1.9.0...v1.9.3)

---
updated-dependencies:
- dependency-name: github.com/sirupsen/logrus
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod" 
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "npm"
+    directory: "/web"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/build.yml

@@ -10,6 +10,7 @@ on:
 permissions:
   contents: write
   packages: write
+  security-events: write
 
 jobs:
   build:
@@ -22,11 +23,17 @@ jobs:
 
     - uses: actions/setup-go@v2
       with:
-        go-version: '1.17'
+        go-version: '1.20'
 
-    - name: Confirm Version
+    - name: Go Version
       run: go version
 
+    - name: Go Vet
+      run: go vet ./...
+
+    - name: Docker Version
+      run: docker version
+
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v2
 
@@ -34,6 +41,7 @@ jobs:
       id: buildx
       uses: docker/setup-buildx-action@v2
 
+# Standard Build
     - name: Build
       uses: goreleaser/goreleaser-action@v2
       if: startsWith(github.ref, 'refs/tags/') == false
@@ -43,23 +51,59 @@ jobs:
         args: release --rm-dist --snapshot
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DOCKER_BUILDKIT: 1 
+        COMPOSE_DOCKER_CLI_BUILD: 1
+      
+    - name: 'Get Previous tag'
+      id: previoustag
+      uses: "WyriHaximus/github-action-get-previous-tag@v1"
 
+    - name: Run Trivy vulnerability scanner
+      uses: aquasecurity/trivy-action@7b7aa264d83dc58691451798b4d117d53d21edfe
+      with:
+        image-ref: 'ghcr.io/jackdallas/premiumizearr:${{ steps.previoustag.outputs.tag }}-amd64'
+        format: 'template'
+        template: '@/contrib/sarif.tpl'
+        output: 'trivy-results.sarif'
+        severity: 'CRITICAL,HIGH'
+
+    - name: Upload Trivy scan results to GitHub Security tab
+      uses: github/codeql-action/upload-sarif@v2
+      with:
+        sarif_file: 'trivy-results.sarif'
+  
+# Release build
     - uses: docker/login-action@v1
       if: startsWith(github.ref, 'refs/tags/')
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
-        
+
     - name: Release
       uses: goreleaser/goreleaser-action@v2
-      if: startsWith(github.ref, 'refs/tags/')
+      if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-rc')
       with:
         distribution: goreleaser
         version: latest
         args: release --rm-dist
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DOCKER_BUILDKIT: 1 
+        COMPOSE_DOCKER_CLI_BUILD: 1
+
+# Pre-Release build
+    - name: Pre-Release
+      uses: goreleaser/goreleaser-action@v2
+      if: startsWith(github.ref, 'refs/tags/') && contains(github.ref, '-rc')
+      with:
+        distribution: goreleaser
+        version: latest
+        args: release --rm-dist -f .prerelease.goreleaser.yaml
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DOCKER_BUILDKIT: 1 
+        COMPOSE_DOCKER_CLI_BUILD: 1
 
     - name: Upload assets
       uses: actions/upload-artifact@v2

.github/workflows/codeql-analysis.yml

@@ -1,47 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'go', 'javascript' ]
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/dependabot-auto-merge.yaml

@@ -0,0 +1,27 @@
+name: Dependabot auto-merge
+on: pull_request
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  dependabot:
+    runs-on: ubuntu-latest
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v1
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Enable auto-merge for Dependabot PRs
+        run: gh pr merge --auto --rebase "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      - name: Auto Approve Dependabot PRs
+        run: gh pr review --approve -b "Dependabot auto approve" "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/dependency-review.yml

@@ -0,0 +1,20 @@
+# Dependency Review Action
+#
+# This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging.
+#
+# Source repository: https://github.com/actions/dependency-review-action
+# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement
+name: 'Dependency Review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v3
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v2

.goreleaser.yaml

@@ -1,3 +1,6 @@
+env:
+  - DOCKER_BUILDKIT=1 
+
 before:
   hooks:
     - go mod tidy
@@ -71,7 +74,7 @@ nfpms:
       
 dockers:
   -
-    use: docker
+    use: buildx
     goos: linux
     goarch: amd64
     image_templates:
@@ -83,9 +86,12 @@ dockers:
     - "--label=org.opencontainers.image.title={{.ProjectName}}"
     - "--label=org.opencontainers.image.revision={{.FullCommit}}"
     - "--label=org.opencontainers.image.version={{.Version}}"
+    - "--label=org.opencontainers.image.source=\"https://github.com/JackDallas/Premiumizearr\""
     - "--platform=linux/amd64"
+    dockerfile: "docker/Dockerfile.amd64"
     extra_files:
     - build/static/
+    - docker/
   -
     use: buildx
     goos: linux
@@ -99,9 +105,12 @@ dockers:
     - "--label=org.opencontainers.image.title={{.ProjectName}}"
     - "--label=org.opencontainers.image.revision={{.FullCommit}}"
     - "--label=org.opencontainers.image.version={{.Version}}"
+    - "--label=org.opencontainers.image.source=\"https://github.com/JackDallas/Premiumizearr\""
     - "--platform=linux/arm64"
+    dockerfile: "docker/Dockerfile.arm64"
     extra_files:
     - build/static/
+    - docker/
   -
     use: buildx
     goos: linux
@@ -116,9 +125,12 @@ dockers:
     - "--label=org.opencontainers.image.title={{.ProjectName}}"
     - "--label=org.opencontainers.image.revision={{.FullCommit}}"
     - "--label=org.opencontainers.image.version={{.Version}}"
+    - "--label=org.opencontainers.image.source=\"https://github.com/JackDallas/Premiumizearr\""
     - "--platform=linux/arm/v7"
+    dockerfile: "docker/Dockerfile.armv7"
     extra_files:
     - build/static/
+    - docker/
 
 docker_manifests:
     # Release variants not created on rc-$i tags

.prerelease.goreleaser.yaml

@@ -0,0 +1,147 @@
+before:
+  hooks:
+    - go mod tidy
+    - make web
+builds:
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+    main: ./cmd/premiumizearrd
+    binary: premiumizearrd
+    goarch:
+      - amd64
+      - arm64
+      - arm
+    goarm:
+      - 7
+    ignore:
+      - goos: windows
+        goarch: arm64
+      - goos: windows
+        goarch: arm
+
+archives:
+  - format_overrides:
+      - goos: windows
+        format: zip
+    wrap_in_directory: true
+    files:
+      - README.md
+      - LICENSE
+      - src: build/*.service
+        dst: ./
+        strip_parent: true
+      - src: build/static/*
+        dst: static
+        strip_parent: true
+
+checksum:
+  name_template: 'checksums.txt'
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+
+nfpms:
+  -
+    package_name: premiumizearr
+    bindir: /opt/premiumizearrd
+    vendor: Jack Dallas.
+    homepage: https://github.com/JackDallas/Premiumizearr
+    maintainer: Dallas <jack-dallas@outlook.com>
+    description: Service to connect premiumize.me to Arr clients.
+    license: GPLv3
+    formats:
+      - deb
+    contents:
+      - src: build/static/*
+        dst: /opt/premiumizearrd/static/
+      - src: init/premiumizearrd.service
+        dst: /etc/systemd/system/premiumizearrd.service
+    scripts:
+      postinstall: "scripts/postinstall.sh"
+      
+dockers:
+  -
+    use: docker
+    goos: linux
+    goarch: amd64
+    image_templates:
+    - "ghcr.io/jackdallas/premiumizearr:{{ .Tag }}-amd64"
+    skip_push: "false"
+    build_flag_templates:
+    - "--pull"
+    - "--label=org.opencontainers.image.created={{.Date}}"
+    - "--label=org.opencontainers.image.title={{.ProjectName}}"
+    - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+    - "--label=org.opencontainers.image.version={{.Version}}"
+    - "--label=org.opencontainers.image.source=\"https://github.com/JackDallas/Premiumizearr\""
+    - "--platform=linux/amd64"
+    dockerfile: "docker/Dockerfile.amd64"
+    extra_files:
+    - build/static/
+    - docker/
+  -
+    use: buildx
+    goos: linux
+    goarch: arm64
+    image_templates:
+    - "ghcr.io/jackdallas/premiumizearr:{{ .Tag }}-arm64"
+    skip_push: "false"
+    build_flag_templates:
+    - "--pull"
+    - "--label=org.opencontainers.image.created={{.Date}}"
+    - "--label=org.opencontainers.image.title={{.ProjectName}}"
+    - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+    - "--label=org.opencontainers.image.version={{.Version}}"
+    - "--label=org.opencontainers.image.source=\"https://github.com/JackDallas/Premiumizearr\""
+    - "--platform=linux/arm64"
+    dockerfile: "docker/Dockerfile.arm64"
+    extra_files:
+    - build/static/
+    - docker/
+  -
+    use: buildx
+    goos: linux
+    goarch: arm
+    goarm: 7
+    image_templates:
+    - "ghcr.io/jackdallas/premiumizearr:{{ .Tag }}-armv7"
+    skip_push: "false"
+    build_flag_templates:
+    - "--pull"
+    - "--label=org.opencontainers.image.created={{.Date}}"
+    - "--label=org.opencontainers.image.title={{.ProjectName}}"
+    - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+    - "--label=org.opencontainers.image.version={{.Version}}"
+    - "--label=org.opencontainers.image.source=\"https://github.com/JackDallas/Premiumizearr\""
+    - "--platform=linux/arm/v7"
+    dockerfile: "docker/Dockerfile.armv7"
+    extra_files:
+    - build/static/
+    - docker/
+
+docker_manifests:
+  - skip_push: false
+  - name_template: 'ghcr.io/jackdallas/premiumizearr:dev'
+    image_templates:
+    - 'ghcr.io/jackdallas/premiumizearr:{{ .Tag }}-amd64'
+    - 'ghcr.io/jackdallas/premiumizearr:{{ .Tag }}-armv7'
+    - 'ghcr.io/jackdallas/premiumizearr:{{ .Tag }}-arm64'
+  - name_template: 'ghcr.io/jackdallas/premiumizearr:{{ .Tag }}'
+    image_templates:
+    - 'ghcr.io/jackdallas/premiumizearr:{{ .Tag }}-amd64'
+    - 'ghcr.io/jackdallas/premiumizearr:{{ .Tag }}-armv7'
+    - 'ghcr.io/jackdallas/premiumizearr:{{ .Tag }}-arm64'
+
+release:
+  prerelease: true
+  header: "Premiumizearr Pre-Release {{ .Tag }}"
+  footer: "**Full Changelog**: https://github.com/JackDallas/Premiumizearr/compare/{{ .PreviousTag }}...{{ .Tag }}"
+
+

README.md

@@ -1,5 +1,7 @@
 # Premiumizearr
 
+## Build
+
 [![Build](https://github.com/JackDallas/Premiumizearr/actions/workflows/build.yml/badge.svg)](https://github.com/JackDallas/Premiumizearr/actions/workflows/build.yml)
 
 ## Features
@@ -18,21 +20,37 @@
 
 ### Binary
 
-```
+#### System Install
+
+```cli
 wget https://github.com/JackDallas/Premiumizearr/releases/download/x.x.x/Premiumizearr_x.x.x_linux_amd64.tar.gz
 tar xf Premiumizearr_x.x.x.x_linux_amd64.tar.gz
 cd Premiumizearr_x.x.x.x_linux_amd64
 sudo mkdir /opt/premiumizearrd/
 sudo cp -r premiumizearrd static/ /opt/premiumizearrd/
-sudo cp premiumizearrd /etc/systemd/system/
+sudo cp premiumizearrd.service /etc/systemd/system/
 sudo systemctl-reload
 sudo systemctl enable premiumizearrd.service
 sudo systemctl start premiumizearrd.service
 ```
 
+#### User Install
+
+```cli
+wget https://github.com/JackDallas/Premiumizearr/releases/download/x.x.x/Premiumizearr_x.x.x_linux_amd64.tar.gz
+tar xf Premiumizearr_x.x.x.x_linux_amd64.tar.gz
+cd Premiumizearr_x.x.x.x_linux_amd64
+mkdir -p ~/.local/bin/
+cp -r premiumizearrd static/ ~/.local/bin/
+echo -e "export PATH=~/.local/bin/:$PATH" >> ~/.bashrc 
+source ~/.bashrc
+```
+
+You're now able to run the daemon from anywhere just by typing `premiumizearrd`
+
 ### deb file
 
-```
+```cmd
 wget https://github.com/JackDallas/Premiumizearr/releases/download/x.x.x/premiumizearr_x.x.x._linux_amd64.deb
 sudo dpkg -i premiumizearr_x.x.x.x_linux_amd64.deb
 ```
@@ -41,7 +59,16 @@ sudo dpkg -i premiumizearr_x.x.x.x_linux_amd64.deb
 
 [Docker images are listed here](https://github.com/jackdallas/Premiumizearr/pkgs/container/premiumizearr)
 
-`docker run -p 8182:8182 -v /host/data/path:/data -v /host/downloads/path:/downloads -v /host/blackhole/path:/blackhole ghcr.io/jackdallas/premiumizearr:latest`
+```cmd
+docker run \
+    -v /home/dallas/test/data:/data \
+    -v /home/dallas/test/blackhole:/blackhole \
+    -v /home/dallas/test/downloads:/downloads \
+    -p 8182:8182 \
+    ghcr.io/jackdallas/premiumizearr:latest
+```
+
+If you wish to increase logging (which you'll be asked to do if you submit an issue) you can add `-e PREMIUMIZEARR_LOG_LEVEL=trace` to the command
 
 > Note: The /data mount is where the `config.yaml` and log files are kept
 
@@ -49,11 +76,11 @@ sudo dpkg -i premiumizearr_x.x.x.x_linux_amd64.deb
 
 ### Premiumizearrd
 
-Running for the first time the server will start on http://0.0.0.0:8182
+Running for the first time the server will start on `http://0.0.0.0:8182`
 
-If you already use this binding you can edit them in the `config.yaml` 
+If you already use this binding for something else you can edit them in the `config.yaml`
 
-> Note: Currently most changes in the config ui will not be used until a restart is complete
+> WARNING: This app exposes api keys in the ui and does not have authentication, it is strongly recommended you put it behind a reverse proxy with auth and set the host to `127.0.0.1` to hide the app from the web.
 
 ### Sonarr/Radarr
 
@@ -65,9 +92,11 @@ If you already use this binding you can edit them in the `config.yaml`
 
 ### Reverse Proxy
 
+Premiumizearr does not have authentication built in so it's strongly recommended you use a reverse proxy
+
 #### Nginx
 
-```
+```nginx
 location /premiumizearr/ {
     proxy_pass http://127.0.0.1:8182/;
     proxy_set_header Host $proxy_host;
@@ -79,4 +108,4 @@ location /premiumizearr/ {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $http_connection;
 }
-```
+```

cmd/premiumizearrd/app.go

@@ -18,18 +18,20 @@ type App struct {
 	directoryWatcher   service.DirectoryWatcherService
 	webServer          service.WebServerService
 	arrsManager        service.ArrsManagerService
+	downloadManager    service.DownloadManagerService
+	taskRunner         service.TaskRunnerService
 }
 
 // Makes go vet error - prevents copies
 func (app *App) Lock()   {}
 func (app *App) UnLock() {}
 
-//Start
+// Start
 func (app *App) Start(logLevel string, configFile string, loggingDirectory string) error {
-	//Setup static loggin
+	//Setup static login
 	lvl, err := log.ParseLevel(logLevel)
 	if err != nil {
-		log.Errorf("Error flag not recognized, defaulting to Info!!", err)
+		log.Errorf("Error flag not recognized, defaulting to Info!! %v", err)
 		lvl = log.InfoLevel
 	}
 	log.SetLevel(lvl)
@@ -73,44 +75,47 @@ func (app *App) Start(logLevel string, configFile string, loggingDirectory strin
 	log.Info("---------- Starting premiumizearr daemon ----------")
 	log.Info("")
 
+	log.Trace("Running load or create config")
+	log.Tracef("Reading config file location from flag or env: %s", configFile)
 	app.config, err = config.LoadOrCreateConfig(configFile, app.ConfigUpdatedCallback)
 
 	if err != nil {
 		panic(err)
 	}
 
-	if app.config.PremiumizemeAPIKey == "" {
-		log.Warn("Premiumizeme API key not set, application will not work until it's set")
-	}
-
 	// Initialisation
-
 	app.premiumizemeClient = premiumizeme.NewPremiumizemeClient(app.config.PremiumizemeAPIKey)
 
 	app.transferManager = service.TransferManagerService{}.New()
 	app.directoryWatcher = service.DirectoryWatcherService{}.New()
 	app.webServer = service.WebServerService{}.New()
 	app.arrsManager = service.ArrsManagerService{}.New()
+	app.downloadManager = service.DownloadManagerService{}.New()
+	app.taskRunner = service.TaskRunnerService{}.New()
 
 	// Initialise Services
+	app.taskRunner.Init(&app.config)
+
+	// Must come after taskRunner initialised
 	app.arrsManager.Init(&app.config)
 	app.directoryWatcher.Init(&app.premiumizemeClient, &app.config)
+	app.downloadManager.Init(&app.premiumizemeClient, &app.taskRunner, &app.config)
 
-	// Must come afer arrsManager
+	// Must come after arrsManager
 	app.transferManager.Init(&app.premiumizemeClient, &app.arrsManager, &app.config)
-	// Must come after transfer, arrmanager and directory
+	// Must come after transfer, arrManager and directory
 	app.webServer.Init(&app.transferManager, &app.directoryWatcher, &app.arrsManager, &app.config)
 
+	app.arrsManager.Start()
 	app.webServer.Start()
 	app.directoryWatcher.Start()
+	app.taskRunner.Start()
 	//Block until the program is terminated
 	app.transferManager.Run(15 * time.Second)
 
 	return nil
 }
 
-//ConfigUpdate - Takes copies of current and new configs to prevent editing the original config
-//... well I hope it does but I need to double check how structs are default passed
 func (app *App) ConfigUpdatedCallback(currentConfig config.Config, newConfig config.Config) {
 	app.transferManager.ConfigUpdatedCallback(currentConfig, newConfig)
 	app.directoryWatcher.ConfigUpdatedCallback(currentConfig, newConfig)

config.yaml

@@ -1,17 +1,20 @@
 PremiumizemeAPIKey: xxxxxxxxx
 Arrs:
-- Name: ""
+- Name: Sonarr
   URL: http://localhost:8989
   APIKey: xxxxxxxxx
   Type: Sonarr
-- Name: ""
+- Name: Radarr
   URL: http://localhost:7878
   APIKey: xxxxxxxxx
   Type: Radarr
 BlackholeDirectory: ""
+PollBlackholeDirectory: false
+PollBlackholeIntervalMinutes: 10
 DownloadsDirectory: ""
 UnzipDirectory: ""
 bindIP: 0.0.0.0
 bindPort: "8182"
 WebRoot: ""
 SimultaneousDownloads: 5
+ArrHistoryUpdateIntervalSeconds: 20

docker/Dockerfile.amd64

@@ -0,0 +1,5 @@
+# syntax=edrevo/dockerfile-plus
+
+FROM ghcr.io/linuxserver/baseimage-alpine:3.16-f525477c-ls6@sha256:c25011f564093f523b1a793658d19275d9eac5a7f21aa5d00ce6cdff29c2a8c1
+
+INCLUDE+ docker/Dockerfile.common

docker/Dockerfile.arm64

@@ -0,0 +1,5 @@
+# syntax=edrevo/dockerfile-plus
+
+FROM ghcr.io/linuxserver/baseimage-alpine:3.16-f525477c-ls6@sha256:611bc4a5a75132914dba740dffa4adcea5039fbe67e3704afd5731a55bf8c82f
+
+INCLUDE+ docker/Dockerfile.common

docker/Dockerfile.armv7

@@ -0,0 +1,5 @@
+# syntax=edrevo/dockerfile-plus
+
+FROM ghcr.io/linuxserver/baseimage-alpine:3.16-f525477c-ls6@sha256:a31127cd9764c95d6137764a1854402d3a33ee085edd139e08726e2fc98d2254
+
+INCLUDE+ docker/Dockerfile.common

docker/Dockerfile.common

@@ -1,23 +1,23 @@
-FROM ubuntu:latest
+LABEL build_version="Premiumizearr version:- ${VERSION} Build-date:- ${BUILD_DATE}"
+LABEL maintainer="JackDallas"
 
-RUN apt update && \
-    apt install openssl -y && \
-    apt install ca-certificates \
-    && rm -rf /var/lib/apt/lists/*
+COPY docker/root/ /
+
+EXPOSE 8182
 
 RUN mkdir /data
 RUN mkdir /unzip
 RUN mkdir /downloads
 RUN mkdir /transfers
+RUN mkdir /blackhole
+RUN mkdir -p /opt/app/
+
+WORKDIR /opt/app/
 
 ENV PREMIUMIZEARR_CONFIG_DIR_PATH=/data
 ENV PREMIUMIZEARR_LOGGING_DIR_PATH=/data
 
-EXPOSE 8182
-
-WORKDIR /opt/app/
-
 COPY premiumizearrd /opt/app/
 COPY build/static /opt/app/static
 
-ENTRYPOINT [ "/opt/app/premiumizearrd" ]
+ENTRYPOINT ["/init"]

docker/root/etc/cont-init.d/30-config

@@ -0,0 +1,10 @@
+#!/usr/bin/with-contenv bash
+
+# permissions
+chown -R abc:abc \
+    /data \
+    /unzip \
+    /downloads \
+    /transfers \
+    /blackhole \
+    /opt \

docker/root/etc/services.d/premiumizearr/run

@@ -0,0 +1,6 @@
+#!/usr/bin/with-contenv bash
+
+cd /opt/app/ || exit
+
+exec \
+    s6-setuidgid abc /opt/app/premiumizearrd

go.mod

@@ -1,20 +1,20 @@
 module github.com/jackdallas/premiumizearr
 
-go 1.17
+go 1.20
 
 require (
-	github.com/dustin/go-humanize v1.0.0
-	github.com/fsnotify/fsnotify v1.5.1
+	github.com/dustin/go-humanize v1.0.1
+	github.com/fsnotify/fsnotify v1.6.0
 	github.com/gorilla/mux v1.8.0
 	github.com/orandin/lumberjackrus v1.0.1
-	github.com/sirupsen/logrus v1.8.1
-	golift.io/starr v0.13.0
+	github.com/sirupsen/logrus v1.9.3
+	golift.io/starr v0.14.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/BurntSushi/toml v1.0.0 // indirect
-	golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d // indirect
-	golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 )

go.sum

@@ -1,11 +1,13 @@
 github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
 github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
-github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
@@ -13,46 +15,63 @@ github.com/orandin/lumberjackrus v1.0.1 h1:7ysDQ0MHD79zIFN9/EiDHjUcgopNi5ehtxFDy
 github.com/orandin/lumberjackrus v1.0.1/go.mod h1:xYLt6H8W93pKnQgUQaxsApS0Eb4BwHLOkxk5DVzf5H0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
-github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ=
-golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golift.io/starr v0.13.0 h1:LoihBAH3DQ0ikPNHTVg47tUU+475mzbr1ahMcY5gdno=
-golift.io/starr v0.13.0/go.mod h1:IZIzdT5/NBdhM08xAEO5R1INgGN+Nyp4vCwvgHrbKVs=
+golift.io/starr v0.14.0 h1:G6bmXs0BNS0Kkwhv46FinlW09G6VILV+P6o62SPp2lY=
+golift.io/starr v0.14.0/go.mod h1:LpR7iazinHYn50wNcTkJeVYxbBYQbkU/DcVYBwc5D9I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

init/premiumizearrd.service

@@ -6,6 +6,7 @@ User=1000
 Group=1000
 UMask=0002
 Type=simple
+Environment=PREMIUMIZEARR_LOG_LEVEL=info
 ExecStart=/opt/premiumizearrd/premiumizearrd
 WorkingDirectory=/opt/premiumizearrd/
 TimeoutStopSec=20

internal/arr/radarr.go

@@ -2,13 +2,10 @@ package arr
 
 import (
 	"fmt"
-	"math"
 	"time"
 
-	"github.com/jackdallas/premiumizearr/internal/utils"
 	"github.com/jackdallas/premiumizearr/pkg/premiumizeme"
 	log "github.com/sirupsen/logrus"
-	"golift.io/starr"
 	"golift.io/starr/radarr"
 )
 
@@ -18,7 +15,7 @@ import (
 
 //Data Access
 
-//GetHistory: Updates the history if it's been more than 15 seconds since last update
+// GetHistory: Updates the history if it's been more than 15 seconds since last update
 func (arr *RadarrArr) GetHistory() (radarr.History, error) {
 	arr.LastUpdateMutex.Lock()
 	defer arr.LastUpdateMutex.Unlock()
@@ -29,38 +26,19 @@ func (arr *RadarrArr) GetHistory() (radarr.History, error) {
 	arr.LastUpdateCountMutex.Lock()
 	defer arr.LastUpdateCountMutex.Unlock()
 
-	if time.Since(arr.LastUpdate) > 60*time.Second || arr.History == nil {
-		//Get first page of records
-		his, err := arr.Client.GetHistoryPage(&starr.Req{PageSize: 250, Page: 1})
+	if time.Since(arr.LastUpdate) > time.Duration(arr.Config.ArrHistoryUpdateIntervalSeconds)*time.Second || arr.History == nil {
+		his, err := arr.Client.GetHistory(0, 1000)
 		if err != nil {
-			return radarr.History{}, fmt.Errorf("failed to get history from radarr: %+v", err)
-		}
-
-		if his.TotalRecords == arr.LastUpdateCount && his.TotalRecords > 0 {
-			return *arr.History, nil
-		}
-
-		if his.TotalRecords > 250 {
-			cachedPages := int(math.Ceil(float64(arr.LastUpdateCount) / 250))
-			log.Tracef("Loaded %d cached pages of history\n", cachedPages)
-			remotePages := int(math.Ceil(float64(his.TotalRecords) / float64(250)))
-			log.Tracef("Found %d pages of history on the radarr server\n", cachedPages)
-			for i := 2; i <= remotePages-cachedPages; i++ {
-				log.Tracef("Radarr.GetHistory(): Getting History Page %d", i)
-				h, err := arr.Client.GetHistoryPage(&starr.Req{PageSize: 250, Page: i})
-				if err != nil {
-					return radarr.History{}, fmt.Errorf("failed to get history from radarr: %+v", err)
-				}
-				his.Records = append(his.Records, h.Records...)
-			}
+			return radarr.History{}, err
 		}
 
 		arr.History = his
 		arr.LastUpdate = time.Now()
 		arr.LastUpdateCount = his.TotalRecords
+		log.Debugf("[Radarr] [%s]: Updated history, next update in %d seconds", arr.Name, arr.Config.ArrHistoryUpdateIntervalSeconds)
 	}
 
-	log.Tracef("Radarr.GetHistory(): Returning from GetHistory")
+	log.Tracef("[Radarr] [%s]: Returning from GetHistory", arr.Name)
 	return *arr.History, nil
 }
 
@@ -78,20 +56,18 @@ func (arr *RadarrArr) GetArrName() string {
 //Functions
 
 func (arr *RadarrArr) HistoryContains(name string) (int64, bool) {
-	log.Tracef("Radarr.HistoryContains(): Checking history for %s", name)
+	log.Tracef("Radarr [%s]: Checking history for %s", arr.Name, name)
 	his, err := arr.GetHistory()
 	if err != nil {
-		log.Errorf("Radarr.HistoryContains(): Failed to get history: %+v", err)
+		log.Errorf("Radarr [%s]: Failed to get history: %+v", arr.Name, err)
 		return -1, false
 	}
-	log.Trace("Radarr.HistoryContains(): Got History, now Locking History")
+	log.Tracef("Radarr [%s]: Got History, now Locking History", arr.Name)
 	arr.HistoryMutex.Lock()
 	defer arr.HistoryMutex.Unlock()
 
-	name = utils.StripDownloadTypesExtention(name)
-	// name = strings.ReplaceAll(name, ".", " ")
 	for _, item := range his.Records {
-		if item.SourceTitle == name {
+		if CompareFileNamesFuzzy(item.SourceTitle, name) {
 			return item.ID, true
 		}
 	}

internal/arr/sonarr.go

@@ -2,13 +2,10 @@ package arr
 
 import (
 	"fmt"
-	"math"
 	"time"
 
-	"github.com/jackdallas/premiumizearr/internal/utils"
 	"github.com/jackdallas/premiumizearr/pkg/premiumizeme"
 	log "github.com/sirupsen/logrus"
-	"golift.io/starr"
 	"golift.io/starr/sonarr"
 )
 
@@ -18,7 +15,7 @@ import (
 
 //Data Access
 
-//GetHistory: Updates the history if it's been more than 15 seconds since last update
+// GetHistory: Updates the history if it's been more than 15 seconds since last update
 func (arr *SonarrArr) GetHistory() (sonarr.History, error) {
 	arr.LastUpdateMutex.Lock()
 	defer arr.LastUpdateMutex.Unlock()
@@ -29,38 +26,19 @@ func (arr *SonarrArr) GetHistory() (sonarr.History, error) {
 	arr.LastUpdateCountMutex.Lock()
 	defer arr.LastUpdateCountMutex.Unlock()
 
-	if time.Since(arr.LastUpdate) > 60*time.Second || arr.History == nil {
-		//Get first page of records
-		his, err := arr.Client.GetHistoryPage(&starr.Req{PageSize: 250, Page: 1})
+	if time.Since(arr.LastUpdate) > time.Duration(arr.Config.ArrHistoryUpdateIntervalSeconds)*time.Second || arr.History == nil {
+		his, err := arr.Client.GetHistory(0, 1000)
 		if err != nil {
-			return sonarr.History{}, fmt.Errorf("failed to get history from sonarr: %+v", err)
-		}
-
-		if his.TotalRecords == arr.LastUpdateCount && his.TotalRecords > 0 {
-			return *arr.History, nil
-		}
-
-		if his.TotalRecords > 250 {
-			cachedPages := int(math.Ceil(float64(arr.LastUpdateCount) / 250))
-			fmt.Printf("Loaded %d cached pages of history\n", cachedPages)
-			remotePages := int(math.Ceil(float64(his.TotalRecords) / float64(250)))
-			fmt.Printf("Found %d pages of history on the sonarr server\n", cachedPages)
-			for i := 2; i <= remotePages-cachedPages; i++ {
-				log.Tracef("Sonarr.GetHistory(): Getting History Page %d", i)
-				h, err := arr.Client.GetHistoryPage(&starr.Req{PageSize: 250, Page: i})
-				if err != nil {
-					return sonarr.History{}, fmt.Errorf("failed to get history from sonarr: %+v", err)
-				}
-				his.Records = append(his.Records, h.Records...)
-			}
+			return sonarr.History{}, err
 		}
 
 		arr.History = his
 		arr.LastUpdate = time.Now()
 		arr.LastUpdateCount = his.TotalRecords
+		log.Debugf("[Sonarr] [%s]: Updated history, next update in %d seconds", arr.Name, arr.Config.ArrHistoryUpdateIntervalSeconds)
 	}
 
-	log.Tracef("Sonarr.GetHistory(): Returning from GetHistory")
+	log.Tracef("[Sonarr] [%s]: Returning from GetHistory", arr.Name)
 	return *arr.History, nil
 }
 
@@ -77,22 +55,21 @@ func (arr *SonarrArr) GetArrName() string {
 // Functions
 
 func (arr *SonarrArr) HistoryContains(name string) (int64, bool) {
-	log.Tracef("Sonarr.HistoryContains(): Checking history for %s", name)
+	log.Tracef("Sonarr [%s]: Checking history for %s", arr.Name, name)
 	his, err := arr.GetHistory()
 	if err != nil {
 		return 0, false
 	}
-	log.Trace("Sonarr.HistoryContains(): Got History, now Locking History")
+	log.Tracef("Sonarr [%s]: Got History, now Locking History", arr.Name)
 	arr.HistoryMutex.Lock()
 	defer arr.HistoryMutex.Unlock()
 
-	name = utils.StripDownloadTypesExtention(name)
 	for _, item := range his.Records {
-		if utils.StripDownloadTypesExtention(item.SourceTitle) == name {
+		if CompareFileNamesFuzzy(item.SourceTitle, name) {
 			return item.ID, true
 		}
 	}
-	log.Tracef("Sonarr.HistoryContains(): %s Not in History", name)
+	log.Tracef("Sonarr [%s]: %s Not in History", arr.Name, name)
 
 	return -1, false
 }

internal/arr/types.go

@@ -1,14 +1,43 @@
 package arr
 
 import (
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/jackdallas/premiumizearr/internal/config"
+	"github.com/jackdallas/premiumizearr/internal/utils"
 	"github.com/jackdallas/premiumizearr/pkg/premiumizeme"
 	"golift.io/starr/radarr"
 	"golift.io/starr/sonarr"
 )
 
+func CompareFileNamesFuzzy(a, b string) bool {
+	//Strip file extension
+	a = utils.StripDownloadTypesExtension(a)
+	b = utils.StripDownloadTypesExtension(b)
+	//Strip media type extension
+	a = utils.StripMediaTypesExtension(a)
+	b = utils.StripMediaTypesExtension(b)
+	//Strip Spaces
+	a = strings.ReplaceAll(a, " ", "")
+	b = strings.ReplaceAll(b, " ", "")
+	//Strip periods
+	a = strings.ReplaceAll(a, ".", "")
+	b = strings.ReplaceAll(b, ".", "")
+	//Strip dashes
+	a = strings.ReplaceAll(a, "-", "")
+	b = strings.ReplaceAll(b, "-", "")
+	//Strip underscores
+	a = strings.ReplaceAll(a, "_", "")
+	b = strings.ReplaceAll(b, "_", "")
+	//Convert to lowercase
+	a = strings.ToLower(a)
+	b = strings.ToLower(b)
+
+	return a == b
+}
+
 type IArr interface {
 	HistoryContains(string) (int64, bool)
 	MarkHistoryItemAsFailed(int64) error
@@ -26,6 +55,7 @@ type SonarrArr struct {
 	LastUpdate           time.Time
 	LastUpdateCount      int
 	LastUpdateCountMutex sync.Mutex
+	Config               *config.Config
 }
 
 type RadarrArr struct {
@@ -38,4 +68,5 @@ type RadarrArr struct {
 	LastUpdate           time.Time
 	LastUpdateCount      int
 	LastUpdateCountMutex sync.Mutex
+	Config               *config.Config
 }

internal/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"errors"
 	"io/ioutil"
 
 	"github.com/jackdallas/premiumizearr/internal/utils"
@@ -15,117 +16,202 @@ import (
 // LoadOrCreateConfig - Loads the config from disk or creates a new one
 func LoadOrCreateConfig(altConfigLocation string, _appCallback AppCallback) (Config, error) {
 	config, err := loadConfigFromDisk(altConfigLocation)
+
 	if err != nil {
 		if err == ErrFailedToFindConfigFile {
-			config = defaultConfig(altConfigLocation)
 			log.Warn("No config file found, created default config file")
-			// Override config data directories if running in docker
-			if utils.IsRunningInDockerContainer() {
-				if config.BlackholeDirectory == "" {
-					config.BlackholeDirectory = "/blackhole"
-				}
-				if config.DownloadsDirectory == "" {
-					config.DownloadsDirectory = "/downloads"
-				}
-			}
-			config.Save()
+			config = defaultConfig()
 		}
-		if err == ErrInvalidConfigFile {
-			return config, ErrInvalidConfigFile
+		if err == ErrInvalidConfigFile || err == ErrFailedToSaveConfig {
+			return config, err
 		}
 	}
-	// Override unzip directory if running in docker
-	if utils.IsRunningInDockerContainer() {
-		config.UnzipDirectory = "/unzip"
-	}
+
+	// Override unzip directory if running in docker
+	if utils.IsRunningInDockerContainer() {
+		log.Info("Running in docker, overriding unzip directory!")
+		config.UnzipDirectory = "/unzip"
+		// Override config data directories if blank
+		if config.BlackholeDirectory == "" {
+			log.Trace("Running in docker, overriding blank directory settings for blackhole directory")
+			config.BlackholeDirectory = "/blackhole"
+		}
+		if config.DownloadsDirectory == "" {
+			log.Trace("Running in docker, overriding blank directory settings for downloads directory")
+			config.DownloadsDirectory = "/downloads"
+		}
+	}
+
+	log.Tracef("Setting config location to %s", altConfigLocation)
 
 	config.appCallback = _appCallback
+	config.altConfigLocation = altConfigLocation
+
+	config.Save()
 
 	return config, nil
 }
 
 // Save - Saves the config to disk
-func (c *Config) Save() bool {
+func (c *Config) Save() error {
 	log.Trace("Marshaling & saving config")
 	data, err := yaml.Marshal(*c)
 	if err != nil {
 		log.Error(err)
-		return false
+		return err
 	}
 
-	log.Tracef("Writing config to %s", path.Join(c.altConfigLocation, "config.yaml"))
-	err = ioutil.WriteFile(path.Join(c.altConfigLocation, "config.yaml"), data, 0644)
+	savePath := "./config.yaml"
+	if c.altConfigLocation != "" {
+		savePath = path.Join(c.altConfigLocation, "config.yaml")
+	}
+
+	log.Tracef("Writing config to %s", savePath)
+	err = ioutil.WriteFile(savePath, data, 0644)
 	if err != nil {
 		log.Errorf("Failed to save config file: %+v", err)
-		return false
+		return err
 	}
 
 	log.Trace("Config saved")
-	return true
+	return nil
 }
 
 func loadConfigFromDisk(altConfigLocation string) (Config, error) {
 	var config Config
-	file, err := ioutil.ReadFile(path.Join(altConfigLocation, "config.yaml"))
+
+	log.Trace("Trying to load config from disk")
+	configLocation := path.Join(altConfigLocation, "config.yaml")
+
+	log.Tracef("Reading config from %s", configLocation)
+	file, err := ioutil.ReadFile(configLocation)
 
 	if err != nil {
+		log.Trace("Failed to find config file")
 		return config, ErrFailedToFindConfigFile
 	}
 
-	err = yaml.Unmarshal(file, &config)
+	log.Trace("Loading to interface")
+	var configInterface map[interface{}]interface{}
+	err = yaml.Unmarshal(file, &configInterface)
 	if err != nil {
+		log.Errorf("Failed to unmarshal config file: %+v", err)
 		return config, ErrInvalidConfigFile
 	}
 
-	config = versionUpdateConfig(config)
+	log.Trace("Unmarshalling to struct")
+	err = yaml.Unmarshal(file, &config)
+	if err != nil {
+		log.Errorf("Failed to unmarshal config file: %+v", err)
+		return config, ErrInvalidConfigFile
+	}
 
-	config.Save()
+	log.Trace("Checking for missing config fields")
+	updated := false
+
+	if configInterface["PollBlackholeDirectory"] == nil {
+		log.Info("PollBlackholeDirectory not set, setting to false")
+		config.PollBlackholeDirectory = false
+		updated = true
+	}
+
+	if configInterface["SimultaneousDownloads"] == nil {
+		log.Info("SimultaneousDownloads not set, setting to 5")
+		config.SimultaneousDownloads = 5
+		updated = true
+	}
+
+	if configInterface["PollBlackholeIntervalMinutes"] == nil {
+		log.Info("PollBlackholeIntervalMinutes not set, setting to 10")
+		config.PollBlackholeIntervalMinutes = 10
+		updated = true
+	}
+
+	if configInterface["ArrHistoryUpdateIntervalSeconds"] == nil {
+		log.Info("ArrHistoryUpdateIntervalSeconds not set, setting to 20")
+		config.ArrHistoryUpdateIntervalSeconds = 20
+		updated = true
+	}
 
 	config.altConfigLocation = altConfigLocation
+
+	if updated {
+		log.Trace("Version updated saving")
+		err = config.Save()
+
+		if err == nil {
+			log.Trace("Config saved")
+			return config, nil
+		} else {
+			log.Errorf("Failed to save config to %s", configLocation)
+			log.Error(err)
+			return config, ErrFailedToSaveConfig
+		}
+	}
+
+	log.Trace("Config loaded")
 	return config, nil
 }
 
-func versionUpdateConfig(config Config) Config {
-	// 1.1.3
-	if config.SimultaneousDownloads == 0 {
-		config.SimultaneousDownloads = 5
-	}
-
-	return config
-}
-
-func defaultConfig(altConfigLocation string) Config {
+func defaultConfig() Config {
 	return Config{
 		PremiumizemeAPIKey: "xxxxxxxxx",
 		Arrs: []ArrConfig{
 			{Name: "Sonarr", URL: "http://localhost:8989", APIKey: "xxxxxxxxx", Type: Sonarr},
 			{Name: "Radarr", URL: "http://localhost:7878", APIKey: "xxxxxxxxx", Type: Radarr},
 		},
-		BlackholeDirectory:    "",
-		DownloadsDirectory:    "",
-		UnzipDirectory:        "",
-		BindIP:                "0.0.0.0",
-		BindPort:              "8182",
-		WebRoot:               "",
-		SimultaneousDownloads: 5,
+		BlackholeDirectory:              "",
+		PollBlackholeDirectory:          false,
+		PollBlackholeIntervalMinutes:    10,
+		DownloadsDirectory:              "",
+		UnzipDirectory:                  "",
+		BindIP:                          "0.0.0.0",
+		BindPort:                        "8182",
+		WebRoot:                         "",
+		SimultaneousDownloads:           5,
+		ArrHistoryUpdateIntervalSeconds: 20,
 	}
 }
 
-func (c *Config) GetTempBaseDir() string {
-	if c.UnzipDirectory != "" {
-		return path.Dir(c.UnzipDirectory)
+var (
+	ErrUnzipDirectorySetToRoot    = errors.New("unzip directory set to root")
+	ErrUnzipDirectoryNotWriteable = errors.New("unzip directory not writeable")
+)
+
+func (c *Config) GetUnzipBaseLocation() (string, error) {
+	if c.UnzipDirectory == "" {
+		log.Tracef("Unzip directory not set, using default: %s", os.TempDir())
+		return path.Join(os.TempDir(), "premiumizearrd"), nil
 	}
-	return path.Join(os.TempDir(), "premiumizearrd")
+
+	if c.UnzipDirectory == "/" || c.UnzipDirectory == "\\" || c.UnzipDirectory == "C:\\" {
+		log.Error("Unzip directory set to root, please set a directory")
+		return "", ErrUnzipDirectorySetToRoot
+	}
+
+	if !utils.IsDirectoryWriteable(c.UnzipDirectory) {
+		log.Errorf("Unzip directory not writeable: %s", c.UnzipDirectory)
+		return c.UnzipDirectory, ErrUnzipDirectoryNotWriteable
+	}
+
+	log.Tracef("Unzip directory set to: %s", c.UnzipDirectory)
+	return c.UnzipDirectory, nil
 }
 
-func (c *Config) GetTempDir() (string, error) {
-	// Create temp dir in os temp location
-	tempDir := c.GetTempBaseDir()
-
-	err := os.MkdirAll(tempDir, os.ModePerm)
+func (c *Config) GetNewUnzipLocation() (string, error) {
+	// Create temp dir in os temp location or unzip-directory
+	tempDir, err := c.GetUnzipBaseLocation()
 	if err != nil {
 		return "", err
 	}
+
+	log.Trace("Creating unzip directory")
+	err = os.MkdirAll(tempDir, os.ModePerm)
+	if err != nil {
+		return "", err
+	}
+
+	log.Trace("Creating generated unzip directory")
 	dir, err := ioutil.TempDir(tempDir, "unzip-")
 	if err != nil {
 		return "", err

internal/config/types.go

@@ -5,12 +5,13 @@ import "errors"
 var (
 	ErrInvalidConfigFile      = errors.New("invalid Config File")
 	ErrFailedToFindConfigFile = errors.New("failed to find config file")
+	ErrFailedToSaveConfig     = errors.New("failed to save config")
 )
 
-//ArrType enum for Sonarr/Radarr
+// ArrType enum for Sonarr/Radarr
 type ArrType string
 
-//AppCallback - Callback for the app to use
+// AppCallback - Callback for the app to use
 type AppCallback func(oldConfig Config, newConfig Config)
 
 const (
@@ -19,29 +20,35 @@ const (
 )
 
 type ArrConfig struct {
-	Name   string  `yaml:"Name"`
-	URL    string  `yaml:"URL"`
-	APIKey string  `yaml:"APIKey"`
-	Type   ArrType `yaml:"Type"`
+	Name   string  `yaml:"Name" json:"Name"`
+	URL    string  `yaml:"URL" json:"URL"`
+	APIKey string  `yaml:"APIKey" json:"APIKey"`
+	Type   ArrType `yaml:"Type" json:"Type"`
 }
 
 type Config struct {
 	altConfigLocation string
 	appCallback       AppCallback
 
-	PremiumizemeAPIKey string `yaml:"PremiumizemeAPIKey"`
+	//PremiumizemeAPIKey string with yaml and json tag
+	PremiumizemeAPIKey string `yaml:"PremiumizemeAPIKey" json:"PremiumizemeAPIKey"`
 
-	Arrs []ArrConfig `yaml:"Arrs"`
+	Arrs []ArrConfig `yaml:"Arrs" json:"Arrs"`
 
-	BlackholeDirectory string `yaml:"BlackholeDirectory"`
-	DownloadsDirectory string `yaml:"DownloadsDirectory"`
+	BlackholeDirectory           string `yaml:"BlackholeDirectory" json:"BlackholeDirectory"`
+	PollBlackholeDirectory       bool   `yaml:"PollBlackholeDirectory" json:"PollBlackholeDirectory"`
+	PollBlackholeIntervalMinutes int    `yaml:"PollBlackholeIntervalMinutes" json:"PollBlackholeIntervalMinutes"`
 
-	UnzipDirectory string `yaml:"UnzipDirectory"`
+	DownloadsDirectory string `yaml:"DownloadsDirectory" json:"DownloadsDirectory"`
 
-	BindIP   string `yaml:"bindIP"`
-	BindPort string `yaml:"bindPort"`
+	UnzipDirectory string `yaml:"UnzipDirectory" json:"UnzipDirectory"`
 
-	WebRoot string `yaml:"WebRoot"`
+	BindIP   string `yaml:"bindIP" json:"BindIP"`
+	BindPort string `yaml:"bindPort" json:"BindPort"`
 
-	SimultaneousDownloads int `yaml:"SimultaneousDownloads"`
+	WebRoot string `yaml:"WebRoot" json:"WebRoot"`
+
+	SimultaneousDownloads int `yaml:"SimultaneousDownloads" json:"SimultaneousDownloads"`
+
+	ArrHistoryUpdateIntervalSeconds int `yaml:"ArrHistoryUpdateIntervalSeconds" json:"ArrHistoryUpdateIntervalSeconds"`
 }

internal/directory_watcher/directory_watcher.go

@@ -20,7 +20,7 @@ func NewDirectoryWatcher(path string, recursive bool, matchFunction func(string)
 
 func (w *WatchDirectory) Watch() error {
 	var err error
-	w.watcher, err = fsnotify.NewWatcher()
+	w.Watcher, err = fsnotify.NewWatcher()
 	if err != nil {
 		return err
 	}
@@ -28,7 +28,7 @@ func (w *WatchDirectory) Watch() error {
 	go func() {
 		for {
 			select {
-			case event, ok := <-w.watcher.Events:
+			case event, ok := <-w.Watcher.Events:
 				if !ok {
 					return
 				}
@@ -37,7 +37,7 @@ func (w *WatchDirectory) Watch() error {
 						w.CallbackFunction(event.Name)
 					}
 				}
-			case _, ok := <-w.watcher.Errors:
+			case _, ok := <-w.Watcher.Errors:
 				if !ok {
 					return
 				}
@@ -51,7 +51,7 @@ func (w *WatchDirectory) Watch() error {
 		return err
 	}
 
-	err = w.watcher.Add(cleanPath)
+	err = w.Watcher.Add(cleanPath)
 	if err != nil {
 		return err
 	}
@@ -60,11 +60,11 @@ func (w *WatchDirectory) Watch() error {
 }
 
 func (w *WatchDirectory) UpdatePath(path string) error {
-	w.watcher.Remove(w.Path)
+	w.Watcher.Remove(w.Path)
 	w.Path = path
-	return w.watcher.Add(w.Path)
+	return w.Watcher.Add(w.Path)
 }
 
 func (w *WatchDirectory) Stop() error {
-	return w.watcher.Close()
+	return w.Watcher.Close()
 }

internal/directory_watcher/types.go

@@ -15,5 +15,5 @@ type WatchDirectory struct {
 	// Callback is the function to call when a file is created that matches with MatchFunction.
 	CallbackFunction func(string)
 	// watcher is the fsnotify watcher.
-	watcher *fsnotify.Watcher
+	Watcher *fsnotify.Watcher
 }

internal/service/arrs_manager_service.go

@@ -27,6 +27,7 @@ func (am *ArrsManagerService) Init(_config *config.Config) {
 
 func (am *ArrsManagerService) Start() {
 	am.arrs = []arr.IArr{}
+	log.Debugf("Starting ArrsManagerService")
 	for _, arr_config := range am.config.Arrs {
 		switch arr_config.Type {
 		case config.Sonarr:
@@ -36,8 +37,10 @@ func (am *ArrsManagerService) Start() {
 				Client:     sonarr.New(c),
 				History:    nil,
 				LastUpdate: time.Now(),
+				Config:     am.config,
 			}
 			am.arrs = append(am.arrs, &wrapper)
+			log.Tracef("Added Sonarr arr: %s", arr_config.Name)
 		case config.Radarr:
 			c := starr.New(arr_config.APIKey, arr_config.URL, 0)
 			wrapper := arr.RadarrArr{
@@ -45,12 +48,15 @@ func (am *ArrsManagerService) Start() {
 				Client:     radarr.New(c),
 				History:    nil,
 				LastUpdate: time.Now(),
+				Config:     am.config,
 			}
 			am.arrs = append(am.arrs, &wrapper)
+			log.Tracef("Added Radarr arr: %s", arr_config.Name)
 		default:
-			log.Error("Unknown arr type: %s, not adding Arr %s", arr_config.Type, arr_config.Name)
+			log.Errorf("Unknown arr type: %s, not adding Arr %s", arr_config.Type, arr_config.Name)
 		}
 	}
+	log.Debugf("Created %d Arrs", len(am.arrs))
 }
 
 func (am *ArrsManagerService) Stop() {

internal/service/directory_watcher_service.go

@@ -1,7 +1,7 @@
 package service
 
 import (
-	"io/ioutil"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
@@ -48,35 +48,26 @@ func (dw *DirectoryWatcherService) ConfigUpdatedCallback(currentConfig config.Co
 	if currentConfig.BlackholeDirectory != newConfig.BlackholeDirectory {
 		log.Info("Blackhole directory changed, restarting directory watcher...")
 		log.Info("Running initial directory scan...")
-		go dw.initialDirectoryScan(dw.config.BlackholeDirectory)
+		go dw.directoryScan(dw.config.BlackholeDirectory)
 		dw.watchDirectory.UpdatePath(newConfig.BlackholeDirectory)
 	}
+
+	if currentConfig.PollBlackholeDirectory != newConfig.PollBlackholeDirectory {
+		log.Info("Poll blackhole directory changed, restarting directory watcher...")
+		dw.Start()
+	}
 }
 
 func (dw *DirectoryWatcherService) GetStatus() string {
 	return dw.status
 }
 
-//TODO (Radarr): accept paths as a parameter, support multiple paths
-//Watch: This is the entrypoint for the directory watcher
+// Start: This is the entrypoint for the directory watcher
 func (dw *DirectoryWatcherService) Start() {
 	log.Info("Starting directory watcher...")
 
 	dw.downloadsFolderID = utils.GetDownloadsFolderIDFromPremiumizeme(dw.premiumizemeClient)
 
-	log.Info("Clearing tmp directory...")
-	tempDir := dw.config.GetTempBaseDir()
-
-	if tempDir == "/" || tempDir == "\\" || tempDir == "C:\\" {
-		panic("Unzip directory is set to system root don't do that it will wipe your system!")
-	}
-
-	err := os.RemoveAll(tempDir)
-	if err != nil {
-		log.Errorf("Error clearing tmp directory %s", tempDir)
-	}
-	os.Mkdir(tempDir, os.ModePerm)
-
 	log.Info("Creating Queue...")
 	dw.Queue = stringqueue.NewStringQueue()
 
@@ -84,27 +75,51 @@ func (dw *DirectoryWatcherService) Start() {
 	go dw.processUploads()
 
 	log.Info("Running initial directory scan...")
-	go dw.initialDirectoryScan(dw.config.BlackholeDirectory)
+	go dw.directoryScan(dw.config.BlackholeDirectory)
 
-	// Build and start a DirectoryWatcher
-	dw.watchDirectory = directory_watcher.NewDirectoryWatcher(dw.config.BlackholeDirectory,
-		false,
-		dw.checkFile,
-		dw.addFileToQueue,
-	)
+	if dw.watchDirectory != nil {
+		log.Info("Stopping directory watcher...")
+		err := dw.watchDirectory.Stop()
+		if err != nil {
+			log.Errorf("Error stopping directory watcher: %s", err)
+		}
+	}
 
-	dw.watchDirectory.Watch()
+	if dw.config.PollBlackholeDirectory {
+		log.Info("Starting directory poller...")
+		go func() {
+			for {
+				if !dw.config.PollBlackholeDirectory {
+					log.Info("Directory poller stopped")
+					break
+				}
+				time.Sleep(time.Duration(dw.config.PollBlackholeIntervalMinutes) * time.Minute)
+				log.Infof("Running directory scan of %s", dw.config.BlackholeDirectory)
+				dw.directoryScan(dw.config.BlackholeDirectory)
+				log.Infof("Scan complete, next scan in %d minutes", dw.config.PollBlackholeIntervalMinutes)
+			}
+		}()
+	} else {
+		log.Info("Starting directory watcher...")
+		dw.watchDirectory = directory_watcher.NewDirectoryWatcher(dw.config.BlackholeDirectory,
+			false,
+			dw.checkFile,
+			dw.addFileToQueue,
+		)
+		dw.watchDirectory.Watch()
+	}
 }
 
-func (dw *DirectoryWatcherService) initialDirectoryScan(p string) {
-	log.Trace("Initial directory scan")
-	files, err := ioutil.ReadDir(p)
+func (dw *DirectoryWatcherService) directoryScan(p string) {
+	log.Trace("Running directory scan")
+	files, err := os.ReadDir(p)
 	if err != nil {
-		log.Errorf("Error with initial directory scan %+v", err)
+		log.Errorf("Error with directory scan %+v", err)
+		return
 	}
 
 	for _, file := range files {
-		go func(file os.FileInfo) {
+		go func(file fs.DirEntry) {
 			file_path := path.Join(p, file.Name())
 			if dw.checkFile(file_path) {
 				dw.addFileToQueue(file_path)
@@ -123,7 +138,7 @@ func (dw *DirectoryWatcherService) checkFile(path string) bool {
 	}
 
 	if fi.IsDir() {
-		log.Errorf("Directory created in blackhole %s ignoring (Warning premiumizearrzed does not look in subfolders!)", path)
+		log.Errorf("Directory created in blackhole %s ignoring (Warning premiumizearrd does not look in subfolders!)", path)
 		return false
 	}
 
@@ -167,7 +182,7 @@ func (dw *DirectoryWatcherService) processUploads() {
 					log.Trace("File already uploaded, removing from Disk")
 					os.Remove(filePath)
 				default:
-					log.Error("Error creating transfer: %s", err)
+					log.Errorf("Error creating transfer: %s", err)
 				}
 			} else {
 				dw.status = "Okay"
@@ -179,7 +194,7 @@ func (dw *DirectoryWatcherService) processUploads() {
 			}
 			time.Sleep(time.Second * time.Duration(sleepTimeSeconds))
 		} else {
-			log.Errorf("Received %s from blackhole Queue. Appears to be an empty path.")
+			log.Error("Received blank string from blackhole Queue.")
 		}
 	}
 }

internal/service/download_manager_service.go

@@ -0,0 +1,166 @@
+package service
+
+import (
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/jackdallas/premiumizearr/internal/config"
+	"github.com/jackdallas/premiumizearr/internal/utils"
+
+	"github.com/jackdallas/premiumizearr/pkg/downloadmanager"
+	"github.com/jackdallas/premiumizearr/pkg/premiumizeme"
+	log "github.com/sirupsen/logrus"
+)
+
+type DownloadManagerService struct {
+	downloadManager    *downloadmanager.DownloadManager
+	taskRunner         *TaskRunnerService
+	premiumizemeClient *premiumizeme.Premiumizeme
+	config             *config.Config
+	downloadingIDs     map[string]bool
+
+	downloadsFolderID string
+}
+
+func (DownloadManagerService) New() DownloadManagerService {
+	return DownloadManagerService{
+		downloadsFolderID: "",
+		downloadManager:   &downloadmanager.DownloadManager{},
+		downloadingIDs:    make(map[string]bool),
+	}
+}
+
+func (manager *DownloadManagerService) Init(_premiumizemeClient *premiumizeme.Premiumizeme, taskRunner *TaskRunnerService, _config *config.Config) {
+	manager.premiumizemeClient = _premiumizemeClient
+	manager.taskRunner = taskRunner
+	manager.config = _config
+
+	manager.downloadsFolderID = utils.GetDownloadsFolderIDFromPremiumizeme(manager.premiumizemeClient)
+	manager.CleanUpUnzipDir()
+
+	log.Info("Starting download manager thread")
+	go manager.downloadManager.Run()
+	log.Info("Creating check premiumize downloads folder task")
+	manager.taskRunner.AddTask("Check Premiumize Downloads Folder", 20*time.Second, manager.TaskCheckPremiumizeDownloadsFolder)
+}
+
+func (manager *DownloadManagerService) CleanUpUnzipDir() {
+	log.Info("Cleaning unzip directory")
+
+	unzipBase, err := manager.config.GetUnzipBaseLocation()
+	if err != nil {
+		log.Errorf("Error getting unzip base location: %s", err.Error())
+		return
+	}
+
+	err = utils.RemoveContents(unzipBase)
+	if err != nil {
+		log.Errorf("Error cleaning unzip directory: %s", err.Error())
+		return
+	}
+
+}
+
+func (manager *DownloadManagerService) ConfigUpdatedCallback(currentConfig config.Config, newConfig config.Config) {
+	if currentConfig.UnzipDirectory != newConfig.UnzipDirectory {
+		manager.CleanUpUnzipDir()
+	}
+}
+
+func (manager *DownloadManagerService) TaskCheckPremiumizeDownloadsFolder() {
+	log.Debug("Running Task CheckPremiumizeDownloadsFolder")
+
+	items, err := manager.premiumizemeClient.ListFolder(manager.downloadsFolderID)
+	if err != nil {
+		log.Errorf("Error listing downloads folder: %s", err.Error())
+		return
+	}
+
+	for _, item := range items {
+		if _, ok := manager.downloadingIDs[item.ID]; ok {
+			continue
+		}
+
+		manager.downloadingIDs[item.ID] = true
+		manager.downloadFinishedTransfer(item, manager.config.DownloadsDirectory)
+	}
+
+}
+
+func (manager *DownloadManagerService) downloadFinishedTransfer(item premiumizeme.Item, downloadDirectory string) {
+	log.Debug("Downloading: ", item.Name)
+	log.Tracef("%+v", item)
+	var link string
+	var err error
+	if item.Type == "file" {
+		link, err = manager.premiumizemeClient.GenerateZippedFileLink(item.ID)
+	} else if item.Type == "folder" {
+		link, err = manager.premiumizemeClient.GenerateZippedFolderLink(item.ID)
+	} else {
+		log.Errorf("Item is not of type 'file' or 'folder' !! Can't download %s", item.Name)
+		return
+	}
+	if err != nil {
+		log.Errorf("Error generating download link: %s", err)
+		return
+	}
+	log.Trace("Downloading from: ", link)
+
+	tempDir, err := manager.config.GetNewUnzipLocation()
+	if err != nil {
+		log.Errorf("Could not create temp dir: %s", err)
+		return
+	}
+
+	splitString := strings.Split(link, "/")
+	savePath := path.Join(tempDir, splitString[len(splitString)-1])
+	log.Trace("Downloading to: ", savePath)
+
+	out, err := os.Create(savePath)
+	if err != nil {
+		log.Errorf("Could not create save path: %s", err)
+		return
+	}
+	defer out.Close()
+
+	transfer, err := manager.downloadManager.AddTransfer(link, savePath)
+	if err != nil {
+		log.Errorf("Could not add transfer: %s", err)
+		return
+	}
+
+	go func() {
+		<-transfer.Finished
+
+		if transfer.GetStatus() == downloadmanager.STATUS_ERROR || transfer.GetStatus() == downloadmanager.STATUS_CANCELED {
+			log.Errorf("Could not download file: %s", strings.Join(transfer.GetErrorStrings(), ", "))
+			return
+		}
+
+		unzipped := true
+		log.Tracef("Unzipping %s to %s", savePath, downloadDirectory)
+		err = utils.Unzip(savePath, downloadDirectory)
+		if err != nil {
+			log.Errorf("Could not unzip file: %s", err)
+			unzipped = false
+		}
+
+		log.Tracef("Removing zip %s from system", savePath)
+		err = os.RemoveAll(savePath)
+		if err != nil {
+			log.Errorf("Could not remove zip: %s", err)
+			return
+		}
+
+		if unzipped {
+			err = manager.premiumizemeClient.DeleteFolder(item.ID)
+			if err != nil {
+				log.Errorf("Error deleting folder on premiumize.me: %s", err)
+				return
+			}
+		}
+
+	}()
+}

internal/service/task_runner_service.go

@@ -0,0 +1,68 @@
+package service
+
+import (
+	"sync"
+	"time"
+
+	"github.com/jackdallas/premiumizearr/internal/config"
+)
+
+type ServiceTask struct {
+	TaskName      string        `json:"task_name"`
+	LastCompleted time.Time     `json:"last_completed"`
+	Interval      time.Duration `json:"interval"`
+	IsRunning     bool          `json:"is_running"`
+	function      func()
+}
+
+type TaskRunnerService struct {
+	tasks      []ServiceTask
+	tasksMutex *sync.RWMutex
+	config     *config.Config
+}
+
+func (TaskRunnerService) New() TaskRunnerService {
+	return TaskRunnerService{
+		tasks:      []ServiceTask{},
+		tasksMutex: &sync.RWMutex{},
+	}
+}
+
+func (manager *TaskRunnerService) Init(config *config.Config) {
+	manager.config = config
+}
+
+func (manager *TaskRunnerService) AddTask(taskName string, interval time.Duration, function func()) {
+	manager.tasksMutex.Lock()
+	defer manager.tasksMutex.Unlock()
+	manager.tasks = append(manager.tasks, ServiceTask{
+		TaskName:      taskName,
+		LastCompleted: time.Time{},
+		Interval:      interval,
+		IsRunning:     false,
+		function:      function,
+	})
+}
+
+func (manager *TaskRunnerService) Start() {
+	go func() {
+		for {
+			manager.tasksMutex.Lock()
+			for _, task := range manager.tasks {
+				if task.IsRunning {
+					continue
+				}
+				if time.Since(task.LastCompleted) > task.Interval {
+					task.IsRunning = true
+					go func(task ServiceTask) {
+						task.function()
+						task.LastCompleted = time.Now()
+						task.IsRunning = false
+					}(task)
+				}
+			}
+			manager.tasksMutex.Unlock()
+			time.Sleep(time.Millisecond * 50)
+		}
+	}()
+}

internal/service/transfer_manager_service.go

@@ -1,25 +1,15 @@
 package service
 
 import (
-	"os"
-	"path"
-	"strings"
-	"sync"
+	"fmt"
 	"time"
 
 	"github.com/jackdallas/premiumizearr/internal/config"
-	"github.com/jackdallas/premiumizearr/internal/progress_downloader"
 	"github.com/jackdallas/premiumizearr/internal/utils"
 	"github.com/jackdallas/premiumizearr/pkg/premiumizeme"
 	log "github.com/sirupsen/logrus"
 )
 
-type DownloadDetails struct {
-	Added              time.Time
-	Name               string
-	ProgressDownloader *progress_downloader.WriteCounter
-}
-
 type TransferManagerService struct {
 	premiumizemeClient *premiumizeme.Premiumizeme
 	arrsManager        *ArrsManagerService
@@ -27,10 +17,7 @@ type TransferManagerService struct {
 	lastUpdated        int64
 	transfers          []premiumizeme.Transfer
 	runningTask        bool
-	downloadListMutex  *sync.Mutex
-	downloadList       map[string]*DownloadDetails
 	status             string
-	downloadsFolderID  string
 }
 
 // Handle
@@ -41,10 +28,7 @@ func (t TransferManagerService) New() TransferManagerService {
 	t.lastUpdated = time.Now().Unix()
 	t.transfers = make([]premiumizeme.Transfer, 0)
 	t.runningTask = false
-	t.downloadListMutex = &sync.Mutex{}
-	t.downloadList = make(map[string]*DownloadDetails, 0)
 	t.status = ""
-	t.downloadsFolderID = ""
 	return t
 }
 
@@ -52,28 +36,40 @@ func (t *TransferManagerService) Init(pme *premiumizeme.Premiumizeme, arrsManage
 	t.premiumizemeClient = pme
 	t.arrsManager = arrsManager
 	t.config = config
+	t.CleanUpUnzipDir()
+}
+
+func (t *TransferManagerService) CleanUpUnzipDir() {
+	log.Info("Cleaning unzip directory")
+
+	unzipBase, err := t.config.GetUnzipBaseLocation()
+	if err != nil {
+		log.Errorf("Error getting unzip base location: %s", err.Error())
+		return
+	}
+
+	err = utils.RemoveContents(unzipBase)
+	if err != nil {
+		log.Errorf("Error cleaning unzip directory: %s", err.Error())
+		return
+	}
+
 }
 
 func (manager *TransferManagerService) ConfigUpdatedCallback(currentConfig config.Config, newConfig config.Config) {
-	//All config access is dynamic currently, function kept for potential future use
+	//NOOP
 }
 
 func (manager *TransferManagerService) Run(interval time.Duration) {
-	manager.downloadsFolderID = utils.GetDownloadsFolderIDFromPremiumizeme(manager.premiumizemeClient)
 	for {
 		manager.runningTask = true
 		manager.TaskUpdateTransfersList()
-		manager.TaskCheckPremiumizeDownloadsFolder()
 		manager.runningTask = false
 		manager.lastUpdated = time.Now().Unix()
 		time.Sleep(interval)
 	}
 }
 
-func (manager *TransferManagerService) GetDownloads() map[string]*DownloadDetails {
-	return manager.downloadList
-}
-
 func (manager *TransferManagerService) GetTransfers() *[]premiumizeme.Transfer {
 	return &manager.transfers
 }
@@ -81,6 +77,10 @@ func (manager *TransferManagerService) GetStatus() string {
 	return manager.status
 }
 
+func (manager *TransferManagerService) updateTransfers(transfers []premiumizeme.Transfer) {
+	manager.transfers = transfers
+}
+
 func (manager *TransferManagerService) TaskUpdateTransfersList() {
 	log.Debug("Running Task UpdateTransfersList")
 	transfers, err := manager.premiumizemeClient.GetTransfers()
@@ -90,6 +90,32 @@ func (manager *TransferManagerService) TaskUpdateTransfersList() {
 	}
 	manager.updateTransfers(transfers)
 
+	log.Tracef("Checking %d transfers against %d Arr clients", len(transfers), len(manager.arrsManager.GetArrs()))
+	earlyReturn := false
+
+	if len(transfers) == 0 {
+		manager.status = "No transfers"
+		earlyReturn = true
+	} else {
+		manager.status = fmt.Sprintf("Got %d transfers", len(transfers))
+	}
+
+	if len(manager.arrsManager.GetArrs()) == 0 {
+		manager.status = fmt.Sprintf("%s, no ARRs available", manager.status)
+		earlyReturn = true
+	}
+	//else {
+	// 	//TODO: Test
+	// 	// if manager.status[len(manager.status)-19:] == ", no ARRs available" {
+	// 	// 	manager.status = manager.status[:len(manager.status)-19]
+	// 	// }
+	// 	fmt.Print(manager.status)
+	// }
+
+	if earlyReturn {
+		return
+	}
+
 	for _, transfer := range transfers {
 		found := false
 		for _, arr := range manager.arrsManager.GetArrs() {
@@ -112,149 +138,3 @@ func (manager *TransferManagerService) TaskUpdateTransfersList() {
 		}
 	}
 }
-
-func (manager *TransferManagerService) TaskCheckPremiumizeDownloadsFolder() {
-	log.Debug("Running Task CheckPremiumizeDownloadsFolder")
-
-	items, err := manager.premiumizemeClient.ListFolder(manager.downloadsFolderID)
-	if err != nil {
-		log.Errorf("Error listing downloads folder: %s", err.Error())
-		return
-	}
-
-	for _, item := range items {
-		if manager.countDownloads() < manager.config.SimultaneousDownloads {
-			log.Debugf("Processing completed item: %s", item.Name)
-			manager.HandleFinishedItem(item, manager.config.DownloadsDirectory)
-		} else {
-			log.Debugf("Not processing any more transfers, %d are running and cap is %d", manager.countDownloads(), manager.config.SimultaneousDownloads)
-			break
-		}
-	}
-}
-
-func (manager *TransferManagerService) updateTransfers(transfers []premiumizeme.Transfer) {
-	manager.transfers = transfers
-}
-
-func (manager *TransferManagerService) addDownload(item *premiumizeme.Item) {
-	manager.downloadListMutex.Lock()
-	defer manager.downloadListMutex.Unlock()
-
-	manager.downloadList[item.Name] = &DownloadDetails{
-		Added:              time.Now(),
-		Name:               item.Name,
-		ProgressDownloader: progress_downloader.NewWriteCounter(),
-	}
-}
-
-func (manager *TransferManagerService) countDownloads() int {
-	manager.downloadListMutex.Lock()
-	defer manager.downloadListMutex.Unlock()
-
-	return len(manager.downloadList)
-}
-
-func (manager *TransferManagerService) removeDownload(name string) {
-	manager.downloadListMutex.Lock()
-	defer manager.downloadListMutex.Unlock()
-
-	delete(manager.downloadList, name)
-}
-
-func (manager *TransferManagerService) downloadExists(itemName string) bool {
-	manager.downloadListMutex.Lock()
-	defer manager.downloadListMutex.Unlock()
-
-	for _, dl := range manager.downloadList {
-		if dl.Name == itemName {
-			return true
-		}
-	}
-
-	return false
-}
-
-// Returns when the download has been added to the list
-func (manager *TransferManagerService) HandleFinishedItem(item premiumizeme.Item, downloadDirectory string) {
-	if manager.downloadExists(item.Name) {
-		log.Tracef("Transfer %s is already downloading", item.Name)
-		return
-	}
-
-	manager.addDownload(&item)
-
-	go func() {
-		log.Debug("Downloading: ", item.Name)
-		log.Tracef("%+v", item)
-		var link string
-		var err error
-		if item.Type == "file" {
-			link, err = manager.premiumizemeClient.GenerateZippedFileLink(item.ID)
-		} else if item.Type == "folder" {
-			link, err = manager.premiumizemeClient.GenerateZippedFolderLink(item.ID)
-		} else {
-			log.Errorf("Item is not of type 'file' or 'folder' !! Can't download %s", item.Name)
-			return
-		}
-		if err != nil {
-			log.Error("Error generating download link: %s", err)
-			manager.removeDownload(item.Name)
-			return
-		}
-		log.Trace("Downloading: ", link)
-
-		tempDir, err := manager.config.GetTempDir()
-		if err != nil {
-			log.Errorf("Could not create temp dir: %s", err)
-			manager.removeDownload(item.Name)
-			return
-		}
-
-		splitString := strings.Split(link, "/")
-		savePath := path.Join(tempDir, splitString[len(splitString)-1])
-		log.Trace("Downloading to: ", savePath)
-
-		out, err := os.Create(savePath)
-		if err != nil {
-			log.Errorf("Could not create save path: %s", err)
-			manager.removeDownload(item.Name)
-			return
-		}
-		defer out.Close()
-
-		err = progress_downloader.DownloadFile(link, savePath, manager.downloadList[item.Name].ProgressDownloader)
-
-		if err != nil {
-			log.Errorf("Could not download file: %s", err)
-			manager.removeDownload(item.Name)
-			return
-		}
-
-		log.Tracef("Unzipping %s to %s", savePath, downloadDirectory)
-		err = utils.Unzip(savePath, downloadDirectory)
-		if err != nil {
-			log.Errorf("Could not unzip file: %s", err)
-			manager.removeDownload(item.Name)
-			return
-		}
-
-		log.Tracef("Removing zip %s from system", savePath)
-		err = os.RemoveAll(savePath)
-		if err != nil {
-			manager.removeDownload(item.Name)
-			log.Errorf("Could not remove zip: %s", err)
-			return
-		}
-
-		err = manager.premiumizemeClient.DeleteFolder(item.ID)
-		if err != nil {
-			manager.removeDownload(item.Name)
-			log.Error("Error deleting folder on premiumuze.me: %s", err)
-			return
-		}
-
-		//Remove download entry from downloads map
-		manager.removeDownload(item.Name)
-	}()
-}

internal/service/web_service_handlers.go

@@ -37,6 +37,7 @@ type BlackholeResponse struct {
 }
 
 type Download struct {
+	ID       int64  `json:"id"`
 	Added    int64  `json:"added"`
 	Name     string `json:"name"`
 	Progress string `json:"progress"`
@@ -53,14 +54,15 @@ func (s *WebServerService) DownloadsHandler(w http.ResponseWriter, r *http.Reque
 	if s.transferManager == nil {
 		resp.Status = "Not Initialized"
 	} else {
-		for _, v := range s.transferManager.GetDownloads() {
-			resp.Downloads = append(resp.Downloads, Download{
-				Added:    v.Added.Unix(),
-				Name:     v.Name,
-				Progress: v.ProgressDownloader.GetProgress(),
-				Speed:    v.ProgressDownloader.GetSpeed(),
-			})
-		}
+		// for _, v := range s.transferManager.GetDownloads() {
+		// 	resp.Downloads = append(resp.Downloads, Download{
+		// 		ID:       v.ID,
+		// 		Added:    v.Added.Unix(),
+		// 		Name:     v.Name,
+		// 		Progress: v.ProgressDownloader.GetProgress(),
+		// 		Speed:    v.ProgressDownloader.GetSpeed(),
+		// 	})
+		// }
 		resp.Status = ""
 	}
 

internal/utils/utils.go

@@ -12,7 +12,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func StripDownloadTypesExtention(fileName string) string {
+func StripDownloadTypesExtension(fileName string) string {
 	var exts = [...]string{".nzb", ".magnet"}
 	for _, ext := range exts {
 		fileName = strings.TrimSuffix(fileName, ext)
@@ -21,6 +21,15 @@ func StripDownloadTypesExtention(fileName string) string {
 	return fileName
 }
 
+func StripMediaTypesExtension(fileName string) string {
+	var exts = [...]string{".mkv", ".mp4", ".avi", ".mov", ".flv", ".wmv", ".mpg", ".mpeg", ".m4v", ".3gp", ".3g2", ".m2ts", ".mts", ".ts", ".webm", ".m4a", ".m4b", ".m4p", ".m4r", ".m4v"}
+	for _, ext := range exts {
+		fileName = strings.TrimSuffix(fileName, ext)
+	}
+
+	return fileName
+}
+
 // https://golangcode.com/unzip-files-in-go/
 func Unzip(src string, dest string) error {
 	r, err := zip.OpenReader(src)
@@ -84,11 +93,11 @@ func StringInSlice(a string, list []string) int {
 
 func GetDownloadsFolderIDFromPremiumizeme(premiumizemeClient *premiumizeme.Premiumizeme) string {
 	var downloadsFolderID string
+
 	folders, err := premiumizemeClient.GetFolders()
 	if err != nil {
 		log.Errorf("Error getting folders: %s", err)
-		log.Errorf("Cannot read folders from premiumize.me, application will not run!")
-		return ""
+		log.Fatal("Cannot read folders from premiumize.me, application will not run!")
 	}
 
 	const folderName = "arrDownloads"
@@ -131,3 +140,43 @@ func IsRunningInDockerContainer() bool {
 
 	return false
 }
+
+func IsDirectoryWriteable(path string) bool {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		log.Errorf("Directory does not exist: %s", path)
+		return false
+	}
+
+	if _, err := os.Create(path + "/test.txt"); err != nil {
+		log.Errorf("Cannot write test.txt to directory: %s", path)
+		return false
+	}
+
+	// Delete test file
+	if err := os.Remove(path + "/test.txt"); err != nil {
+		log.Errorf("Cannot delete test.txt file in: %s", path)
+		return false
+	}
+
+	return true
+}
+
+// https://stackoverflow.com/questions/33450980/how-to-remove-all-contents-of-a-directory-using-golang
+func RemoveContents(dir string) error {
+	d, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	names, err := d.Readdirnames(-1)
+	if err != nil {
+		return err
+	}
+	for _, name := range names {
+		err = os.RemoveAll(filepath.Join(dir, name))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/clouddownloader/interfaces.go

@@ -1,6 +0,0 @@
-package clouddownloader
-
-// Interface for the CloudDownloader interface
-type CloudDownloaderInterface interface {
-	GetTransfers() []Transfer
-}

pkg/clouddownloader/types.go

@@ -1,4 +0,0 @@
-package clouddownloader
-
-type Transfer struct {
-}

pkg/downloadmanager/downloadmanager.go

@@ -0,0 +1,86 @@
+package downloadmanager
+
+import (
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func (d *DownloadManager) Run() {
+	for {
+		select {
+		case <-d.CancelChannel:
+			return
+		default:
+			time.Sleep(time.Millisecond * 100)
+			for i := 0; i < len(d.transfers); i++ {
+				t := &d.transfers[i]
+				switch t.GetStatus() {
+				case STATUS_QUEUED:
+					if d.GetActiveTransferCount() < d.MaxSimultaneousDownloads {
+						if err := t.Download(); err != nil {
+							log.Errorf("Error downloading: %s", err)
+						}
+					} else {
+						log.Debugf("Too many active transfers, skipping %d", t.GetID())
+					}
+					return
+				}
+			}
+		}
+	}
+}
+
+func (d *DownloadManager) GetTransfers() []Transfer {
+	d.transfersLock.Lock()
+	defer d.transfersLock.Unlock()
+	return d.transfers
+}
+
+func (d *DownloadManager) GetTransfer(id int64) (*Transfer, error) {
+	d.transfersLock.Lock()
+	defer d.transfersLock.Unlock()
+	for i := 0; i < len(d.transfers); i++ {
+		if d.transfers[i].GetID() == id {
+			return &d.transfers[i], nil
+		}
+	}
+	return nil, ErrorNoTransferWithID
+}
+
+func (d *DownloadManager) AddTransfer(url string, savePath string) (*Transfer, error) {
+	d.transfersLock.Lock()
+	defer d.transfersLock.Unlock()
+
+	nextID := d.IdCounter.Add(1)
+
+	d.transfers = append(d.transfers, NewTransfer(nextID, url, savePath))
+
+	log.Debugf("Added transfer %d", nextID)
+	return d.GetTransfer(nextID)
+}
+
+func (d *DownloadManager) GetActiveTransferCount() int {
+	c := 0
+
+	for i := 0; i < len(d.transfers); i++ {
+		if d.transfers[i].GetStatus() == STATUS_DOWNLOADING {
+			c++
+		}
+	}
+
+	return c
+}
+
+func (d *DownloadManager) RemoveTransfer(id int64) error {
+	d.transfersLock.Lock()
+	defer d.transfersLock.Unlock()
+
+	for i := range d.transfers {
+		if d.transfers[i].GetID() == id {
+			return d.transfers[i].Cancel()
+		}
+	}
+
+	return ErrorNoTransferWithID
+}

pkg/downloadmanager/progress_downloader.go

@@ -1,4 +1,4 @@
-package progress_downloader
+package downloadmanager
 
 // https://golangcode.com/download-a-file-with-progress/
 
@@ -19,6 +19,7 @@ type WriteCounter struct {
 	LastUpdate time.Time
 	LastAmount uint64
 	Total      uint64
+	Closing    bool
 }
 
 func NewWriteCounter() *WriteCounter {
@@ -41,6 +42,9 @@ func (wc *WriteCounter) GetSpeed() string {
 }
 
 func (wc *WriteCounter) Write(p []byte) (int, error) {
+	if wc.Closing {
+		return 0, io.ErrClosedPipe
+	}
 	n := len(p)
 	wc.LastAmount = wc.Total
 	wc.Total += uint64(n)
@@ -52,9 +56,6 @@ func (wc WriteCounter) GetProgress() string {
 	return fmt.Sprintf("%s complete", humanize.Bytes(wc.Total))
 }
 
-// DownloadFile will download a url to a local file. It's efficient because it will
-// write as it downloads and not load the whole file into memory. We pass an io.TeeReader
-// into Copy() to report progress on the download.
 func DownloadFile(url string, filepath string, counter *WriteCounter) error {
 
 	// Create the file, but give it a tmp file extension, this means we won't overwrite a
@@ -72,6 +73,7 @@ func DownloadFile(url string, filepath string, counter *WriteCounter) error {
 	}
 	defer resp.Body.Close()
 
+	// resp.Body.
 	if _, err = io.Copy(out, io.TeeReader(resp.Body, counter)); err != nil {
 		out.Close()
 		return err

pkg/downloadmanager/transfer.go

@@ -0,0 +1,208 @@
+package downloadmanager
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	ErrorNoTransferWithID = errors.New("no transfer with id")
+)
+
+type transferStatus int
+
+const (
+	STATUS_QUEUED transferStatus = iota
+	STATUS_DOWNLOADING
+	STATUS_PAUSED
+	STATUS_COMPLETED
+	STATUS_CANCELED
+	STATUS_ERROR
+)
+
+type Transfer struct {
+	id               int64
+	totalSize        atomic.Int64
+	downloaded       atomic.Int64
+	savePath         string
+	url              string
+	urlLock          sync.Mutex
+	status           transferStatus
+	statusLock       sync.Mutex
+	errorStrings     []string
+	errorStringsLock sync.Mutex
+	tempFileName     string
+	Finished         chan bool
+}
+
+func NewTransfer(id int64, url string, savePath string) Transfer {
+	return Transfer{
+		id:               id,
+		totalSize:        atomic.Int64{},
+		downloaded:       atomic.Int64{},
+		savePath:         savePath,
+		url:              url,
+		urlLock:          sync.Mutex{},
+		status:           STATUS_QUEUED,
+		statusLock:       sync.Mutex{},
+		errorStrings:     make([]string, 0),
+		errorStringsLock: sync.Mutex{},
+		tempFileName:     "",
+	}
+}
+
+func (t *Transfer) SetID(id int64) {
+	atomic.StoreInt64(&t.id, id)
+}
+
+func (t *Transfer) GetID() int64 {
+	return atomic.LoadInt64(&t.id)
+}
+
+func (t *Transfer) SetTotalSize(size int64) {
+	t.totalSize.Store(size)
+}
+
+func (t *Transfer) GetTotalSize() int64 {
+	return t.totalSize.Load()
+}
+
+func (t *Transfer) SetDownloaded(size int64) {
+	t.downloaded.Store(size)
+}
+
+func (t *Transfer) GetDownloaded() int64 {
+	return t.downloaded.Load()
+}
+
+func (t *Transfer) SetURL(url string) {
+	t.urlLock.Lock()
+	t.url = url
+	t.urlLock.Unlock()
+}
+
+func (t *Transfer) GetURL() string {
+	t.urlLock.Lock()
+	defer t.urlLock.Unlock()
+	return t.url
+}
+
+func (t *Transfer) SetStatus(status transferStatus) {
+	t.statusLock.Lock()
+	t.status = status
+	t.statusLock.Unlock()
+}
+
+func (t *Transfer) GetStatus() transferStatus {
+	t.statusLock.Lock()
+	defer t.statusLock.Unlock()
+	return t.status
+}
+
+func (t *Transfer) AddErrorString(str string) {
+	t.errorStringsLock.Lock()
+	t.errorStrings = append(t.errorStrings, str)
+	t.errorStringsLock.Unlock()
+}
+
+func (t *Transfer) GetErrorStrings() []string {
+	t.errorStringsLock.Lock()
+	defer t.errorStringsLock.Unlock()
+	return t.errorStrings
+}
+
+func (t *Transfer) GetTempFilePath() string {
+	if t.tempFileName == "" {
+		url, err := url.Parse(t.GetURL())
+		if err != nil {
+			t.tempFileName = fmt.Sprintf("download-%d", t.GetID())
+		} else {
+			finalPath := strings.Split(url.Path, "/")[len(strings.Split(url.Path, "/"))-1]
+			t.tempFileName = fmt.Sprintf("download-%d-%s", t.GetID(), finalPath)
+		}
+	}
+
+	return t.tempFileName
+}
+
+func (t *Transfer) Write(p []byte) (int, error) {
+	if t.GetStatus() == STATUS_CANCELED || t.GetStatus() == STATUS_PAUSED {
+		return 0, io.EOF
+	}
+	t.SetDownloaded(t.GetDownloaded() + int64(len(p)))
+	return len(p), nil
+}
+
+func (t *Transfer) Pause() error {
+	t.SetStatus(STATUS_PAUSED)
+	return nil
+}
+
+func (t *Transfer) Cancel() error {
+	t.SetStatus(STATUS_CANCELED)
+	t.Finished <- true
+	return nil
+}
+
+func (t *Transfer) Resume() error {
+	return t.Download()
+}
+
+func (t *Transfer) Download() error {
+	client := &http.Client{}
+
+	//Built http get request with a content range header
+	req, err := http.NewRequest("GET", t.GetURL(), nil)
+	if err != nil {
+		return err
+	}
+	if t.GetDownloaded() > 0 {
+		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", t.GetDownloaded()))
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	t.SetTotalSize(resp.ContentLength)
+	var out *os.File
+
+	if t.GetDownloaded() > 0 {
+		out, err = os.Open(t.GetTempFilePath())
+	} else {
+		out, err = os.Create(t.GetTempFilePath())
+	}
+
+	if err != nil {
+		return err
+	}
+
+	t.SetStatus(STATUS_DOWNLOADING)
+	go func() {
+		defer out.Close()
+
+		if _, err := io.Copy(out, io.TeeReader(resp.Body, t)); err != nil {
+			t.AddErrorString(err.Error())
+			t.SetStatus(STATUS_ERROR)
+			log.Error(err)
+		}
+		t.SetStatus(STATUS_COMPLETED)
+		t.Finished <- true
+	}()
+
+	return nil
+}
+
+func Start() {
+
+}

pkg/downloadmanager/types.go

@@ -0,0 +1,25 @@
+package downloadmanager
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+// type DownloadManager interface {
+// 	GetTransfers() []Transfer
+
+// 	GetTransfer(id int64) (*Transfer, error)
+// 	AddTransfer(url string) (*Transfer, error)
+// 	RemoveTransfer(id int64) error
+// }
+
+type DownloadManager struct {
+	MaxSimultaneousDownloads int
+
+	transfers     []Transfer
+	transfersLock sync.Mutex
+
+	IdCounter atomic.Int64
+
+	CancelChannel chan bool
+}

pkg/premiumizeme/premiumizeme.go

@@ -38,7 +38,15 @@ func (pm *Premiumizeme) createPremiumizemeURL(urlPath string) (url.URL, error) {
 	return *u, nil
 }
 
+var (
+	ErrAPIKeyNotSet = fmt.Errorf("premiumize.me API key not set")
+)
+
 func (pm *Premiumizeme) GetTransfers() ([]Transfer, error) {
+	if pm.APIKey == "" {
+		return nil, ErrAPIKeyNotSet
+	}
+
 	log.Trace("Getting transfers list from premiumize.me")
 	url, err := pm.createPremiumizemeURL("/transfer/list")
 	if err != nil {
@@ -70,6 +78,10 @@ func (pm *Premiumizeme) GetTransfers() ([]Transfer, error) {
 }
 
 func (pm *Premiumizeme) ListFolder(folderID string) ([]Item, error) {
+	if pm.APIKey == "" {
+		return nil, ErrAPIKeyNotSet
+	}
+
 	var ret []Item
 	url, err := pm.createPremiumizemeURL("/folder/list")
 	if err != nil {
@@ -112,6 +124,10 @@ func (pm *Premiumizeme) ListFolder(folderID string) ([]Item, error) {
 }
 
 func (pm *Premiumizeme) GetFolders() ([]Item, error) {
+	if pm.APIKey == "" {
+		return nil, ErrAPIKeyNotSet
+	}
+
 	log.Trace("Getting folder list from premiumize.me")
 	url, err := pm.createPremiumizemeURL("/folder/list")
 	if err != nil {
@@ -119,30 +135,46 @@ func (pm *Premiumizeme) GetFolders() ([]Item, error) {
 	}
 
 	var ret []Item
-	req, _ := http.NewRequest("GET", url.String(), nil)
+	req, err := http.NewRequest("GET", url.String(), nil)
+	if err != nil {
+		return ret, err
+	}
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return ret, err
 	}
 
-	defer resp.Body.Close()
-	res := ListFoldersResponse{}
-	err = json.NewDecoder(resp.Body).Decode(&res)
+	if resp.StatusCode != 200 {
+		return ret, fmt.Errorf("error listing folder: %s (%d)", resp.Status, resp.StatusCode)
+	}
 
-	if res.Status != "success" {
-		return ret, fmt.Errorf("%s", res.Status)
+	defer resp.Body.Close()
+	list_folders_res := ListFoldersResponse{}
+	err = json.NewDecoder(resp.Body).Decode(&list_folders_res)
+	if err != nil {
+		return ret, err
+	}
+
+	if list_folders_res.Status != "success" {
+		fmt.Printf("%+v\n", resp)
+		fmt.Printf("%+v\n", list_folders_res)
+		return ret, fmt.Errorf(list_folders_res.Message)
 	}
 
 	if err != nil {
 		return ret, err
 	}
 
-	log.Tracef("Received %d Folders", len(res.Content))
-	return res.Content, nil
+	log.Tracef("Received %d Folders", len(list_folders_res.Content))
+	return list_folders_res.Content, nil
 }
 
 func (pm *Premiumizeme) CreateTransfer(filePath string, parentID string) error {
+	if pm.APIKey == "" {
+		return ErrAPIKeyNotSet
+	}
+
 	//TODO: handle file size, i.e. incorrect file being saved
 	log.Trace("Opening file: ", filePath)
 	file, err := os.Open(filePath)
@@ -203,6 +235,10 @@ func (pm *Premiumizeme) CreateTransfer(filePath string, parentID string) error {
 }
 
 func (pm *Premiumizeme) DeleteFolder(folderID string) error {
+	if pm.APIKey == "" {
+		return ErrAPIKeyNotSet
+	}
+
 	url, err := pm.createPremiumizemeURL("/folder/delete")
 	if err != nil {
 		return err
@@ -246,6 +282,10 @@ func (pm *Premiumizeme) DeleteFolder(folderID string) error {
 }
 
 func (pm *Premiumizeme) CreateFolder(folderName string) (string, error) {
+	if pm.APIKey == "" {
+		return "", ErrAPIKeyNotSet
+	}
+
 	url, err := pm.createPremiumizemeURL("/folder/create")
 	if err != nil {
 		return "", err
@@ -289,6 +329,10 @@ func (pm *Premiumizeme) CreateFolder(folderName string) (string, error) {
 }
 
 func (pm *Premiumizeme) DeleteTransfer(id string) error {
+	if pm.APIKey == "" {
+		return ErrAPIKeyNotSet
+	}
+
 	url, err := pm.createPremiumizemeURL("/transfer/delete")
 	if err != nil {
 		return err
@@ -437,6 +481,10 @@ func (pm *Premiumizeme) GenerateZippedFolderLink(fileID string) (string, error)
 }
 
 func (pm *Premiumizeme) generateZip(ID string, srcType SRCType) (string, error) {
+	if pm.APIKey == "" {
+		return "", ErrAPIKeyNotSet
+	}
+
 	// Build URL with apikey
 	URL, err := pm.createPremiumizemeURL("/zip/generate")
 	if err != nil {

pkg/premiumizeme/types.go

@@ -49,7 +49,7 @@ type Item struct {
 }
 type FolderItems struct {
 	Status   string `json:"status"`
-	Contant  []Item `json:"content"`
+	Content  []Item `json:"content"`
 	Name     string `json:"name"`
 	ParentID string `json:"parent_id"`
 	FolderID string `json:"folder_id"`

web/package.json

@@ -2,25 +2,25 @@
   "name": "premiumizearr-ui",
   "version": "0.0.1",
   "devDependencies": {
-    "carbon-components-svelte": "^0.64.0",
-    "carbon-icons-svelte": "^11.0.0",
-    "carbon-preprocess-svelte": "^0.9.0",
-    "copy-webpack-plugin": "^9.0.0",
+    "carbon-components-svelte": "^0.73.5",
+    "carbon-icons-svelte": "^11.4.0",
+    "carbon-preprocess-svelte": "^0.9.1",
+    "copy-webpack-plugin": "^11.0.0",
     "cross-env": "^7.0.0",
     "css-loader": "^5.0.0",
     "esbuild-loader": "^2.0.0",
-    "mini-css-extract-plugin": "^1.0.0",
-    "svelte": "^3.0.0",
-    "svelte-loader": "^3.0.0",
-    "webpack": "^5.0.0",
-    "webpack-cli": "^4.0.0",
-    "webpack-dev-server": "^4.0.0"
+    "mini-css-extract-plugin": "^2.7.5",
+    "svelte": "^3.59.1",
+    "svelte-loader": "^3.1.8",
+    "webpack": "^5.87.0",
+    "webpack-cli": "^5.1.4",
+    "webpack-dev-server": "^4.15.1"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production webpack",
     "dev": "webpack serve --static public"
   },
   "dependencies": {
-    "luxon": "^2.0.0"
+    "luxon": "^3.3.0"
   }
 }

web/src/components/APITable.svelte

@@ -2,6 +2,10 @@
   import { DataTable, InlineLoading } from "carbon-components-svelte";
   import { CalculateAPIPath } from "../Utilities/web_root";
 
+  export let sortable = true;
+  export let sortKey;
+  export let sortOrder;
+
   export let totalName = "";
   export let headers = {};
   export let updateTimeSeconds = 10;
@@ -10,7 +14,7 @@
     if (!data) return [];
     return data;
   };
-  
+
   let updating = false;
   let status = "";
   let rows = [];
@@ -56,6 +60,6 @@
     Message: {status}
   </p>
   <p>
-    <DataTable sortable {headers} {rows} />
+    <DataTable {sortKey} {sortOrder} {sortable} {headers} {rows} />
   </p>
 </main>

web/src/pages/Config.svelte

@@ -7,6 +7,8 @@
     Modal,
     FormGroup,
     Dropdown,
+    Form,
+    Checkbox,
   } from "carbon-components-svelte";
   import {
     Save,
@@ -21,6 +23,8 @@
 
   let config = {
     BlackholeDirectory: "",
+    PollBlackholeDirectory: false,
+    PollBlackholeIntervalMinutes: 10,
     DownloadsDirectory: "",
     UnzipDirectory: "",
     BindIP: "",
@@ -167,7 +171,7 @@
       SetTestArr(index, HelpFilled, "secondary", false);
     }, 1000 * seconds);
   }
-  
+
   getConfig();
 </script>
 
@@ -176,6 +180,12 @@
     <Column>
       <h4>*Arr Settings</h4>
       <FormGroup>
+        <TextInput
+          type="number"
+          disabled={inputDisabled}
+          labelText="Arr Update History Interval (seconds)"
+          bind:value={config.ArrHistoryUpdateIntervalSeconds}
+        />
         {#if config.Arrs !== undefined}
           {#each config.Arrs as arr, i}
             <h5>- {arr.Name ? arr.Name : i}</h5>
@@ -261,6 +271,19 @@
           labelText="Blackhole Directory"
           bind:value={config.BlackholeDirectory}
         />
+        <Checkbox
+          disabled={inputDisabled}
+          bind:checked={config.PollBlackholeDirectory}
+          labelText="Poll Blackhole Directory"
+        />
+        <TextInput
+          type="number"
+          disabled={inputDisabled}
+          labelText="Poll Blackhole Interval Minutes"
+          bind:value={config.PollBlackholeIntervalMinutes}
+        />
+      </FormGroup>
+      <FormGroup>
         <TextInput
           disabled={inputDisabled}
           labelText="Download Directory"

web/src/pages/Info.svelte

@@ -112,11 +112,15 @@
         <h3>Downloads</h3>
         <APITable
           headers={[
+            { key : "id", value : "ID" },
             { key: "added", value: "Added" },
             { key: "name", value: "Name" },
             { key: "progress", value: "Progress" },
             { key: "speed", value: "Speed" },
           ]}
+          sortable={false}
+          sortKey={"id"}
+          sortOrder={"desc"}
           updateTimeSeconds={2}
           APIpath="api/downloads"
           zebra={true}