Skip to content

Commit

Permalink
Update All 2.0
Browse files Browse the repository at this point in the history
  • Loading branch information
theypsilon committed Oct 21, 2022
1 parent 40df448 commit 9671fb4
Show file tree
Hide file tree
Showing 81 changed files with 7,843 additions and 125 deletions.
12 changes: 12 additions & 0 deletions .github/pack_launcher.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
# Copyright (c) 2022 José Manuel Barroso Galindo <[email protected]>

set -euo pipefail

if ! gh release list | grep -q "latest" ; then
gh release create "latest" || true
sleep 15s
fi

zip "update_all.zip" update_all.sh
gh release upload "latest" "update_all.zip" --clobber
24 changes: 24 additions & 0 deletions .github/release.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/usr/bin/env bash
# Copyright (c) 2022 José Manuel Barroso Galindo <[email protected]>

set -euo pipefail

git add dont_download2.sh
git commit -m "BOT: New dont_download2.sh" > /dev/null 2>&1 || true
git fetch origin main

set +e
CHANGES="$(git diff main:dont_download2.sh origin/main:dont_download2.sh | sed '/^[+-]export COMMIT/d' | sed '/^+++/d' | sed '/^---/d' | grep '^[+-]' | wc -l)"
set -e

if [ ${CHANGES} -ge 1 ] ; then
echo "There are changes to push."
echo
git push origin main
echo
echo "New dont_download2.sh can be used."
echo "::set-output name=NEW_RELEASE::yes"
else
echo "Nothing to be updated."
echo "::set-output name=NEW_RELEASE::no"
fi
31 changes: 31 additions & 0 deletions .github/workflows/all_tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
name: All Tests

on:
pull_request:
workflow_dispatch:

jobs:
test:
runs-on: ubuntu-20.04

steps:
- name: Install sharutils
run: sudo apt-get install sharutils

- uses: actions/setup-python@v2
with:
python-version: '3.9'

- uses: actions/checkout@v2

- name: Unit Tests
run: cd src && python3 -m unittest discover -s test/unit

- name: Integration Tests
run: cd src && python3 -m unittest discover -s test/integration

- name: System Quick Tests
run: cd src && python3 -m unittest discover -s test/system/quick

- name: System Slow Tests
run: cd src && python3 -m unittest discover -s test/system/slow
40 changes: 40 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
name: Build

on:
push:
branches:
- main
workflow_dispatch:

jobs:
build:
runs-on: ubuntu-20.04

steps:
- name: Install sharutils
run: sudo apt-get install sharutils

- uses: actions/setup-python@v2
with:
python-version: '3.9'

- uses: actions/checkout@v2

- name: Unit Tests
run: cd src && python3 -m unittest discover -s test/unit

- name: Integration Tests
run: cd src && python3 -m unittest discover -s test/integration

- name: System Tests
run: cd src && python3 -m unittest discover -s test/system

- name: Build
run: ./src/build.sh > dont_download2.sh

- name: Release
id: release
run: |
git config --global user.email "[email protected]"
git config --global user.name "The CI/CD Bot"
./.github/release.sh
20 changes: 20 additions & 0 deletions .github/workflows/pack_launcher.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
name: Pack Launcher

on:
push:
paths:
- 'update_all.sh'
workflow_dispatch:

jobs:
pack:
runs-on: ubuntu-20.04

steps:

- uses: actions/checkout@v2

- name: Pack Launcher
run: ./.github/pack_launcher.sh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
downloader.ini
delme
.venv
.idea
__pycache__
mister.ip
59 changes: 19 additions & 40 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,17 @@ This script runs the [__MiSTer Downloader__](https://github.com/MiSTer-devel/Dow
2. __JTCORES for MiSTer__. Downloads many cores made by [Jotego](https://github.com/jotego). You may select between two versions:
* [jtcores](https://github.com/jotego/jtcores_mister). Public cores only. **[Default option]**
* [jtpremium](https://github.com/jotego/jtpremium). Public + patreon-only cores in their latest versions.
4. [theypsilon Unofficial Distribution](https://github.com/theypsilon/Unofficial_Distribution_MiSTer/tree/main). Downloads [some cores](https://github.com/theypsilon/Updater_script_MiSTer_Unofficial/wiki) that are not yet released but are fairly stable. **[Disabled by default]**
5. [LLAPI Folder](https://github.com/MiSTer-LLAPI/LLAPI_folder_MiSTer). Downloads [LLAPI cores](https://github.com/MiSTer-LLAPI/Updater_script_MiSTer/wiki) which are compatible with BlisSTer and [LLAMA](https://github.com/bootsector/LLAMA). **[Disabled by default]**
6. [Arcade Offset](https://github.com/atrac17/Arcade_Offset). Downloads patched arcade games curated by [atrac17](https://github.com/atrac17). **[Disabled by default]**
7. [Names TXT](https://github.com/ThreepwoodLeBrush/Names_MiSTer). Downloads a complete names.txt file curated by [Threepwood](https://github.com/ThreepwoodLeBrush) that enables better core names in the menus. **[Disabled by default]**
3. [Names TXT](https://github.com/ThreepwoodLeBrush/Names_MiSTer). Downloads a complete names.txt file curated by [Threepwood](https://github.com/ThreepwoodLeBrush) that enables better core names in the menus. **[Disabled by default]**
4. [BIOS Database](https://github.com/theypsilon/BiosDB_MiSTer). Downloads bios from https://archive.org/ for your installed systems. **[Disabled by default]**
5. [Arcade ROMs Database](https://github.com/theypsilon/ArcadeROMsDB_MiSTer). Downloads arcade roms from https://archive.org/ that are compatible with MRAs. **[Disabled by default]**
6. **Unofficial Cores**. A selection of curated databases that provide unofficial cores, including:
1. [Coin-Op Collection](https://github.com/atrac17/Coin-Op_Collection). Downloads cores developed by [Darren](https://github.com/va7deo) and [atrac17](https://github.com/atrac17). **[Disabled by default]**
2. [Arcade Offset](https://github.com/atrac17/Arcade_Offset). Downloads patched arcade games curated by [atrac17](https://github.com/atrac17). **[Disabled by default]**
3. [LLAPI Forks Folder](https://github.com/MiSTer-LLAPI/LLAPI_folder_MiSTer). Downloads [LLAPI cores](https://github.com/MiSTer-LLAPI/Updater_script_MiSTer/wiki) which are compatible with BlisSTer and [LLAMA](https://github.com/bootsector/LLAMA). **[Disabled by default]**
8. **Unofficial Scripts**. A selection of curated databases that provide unofficial scripts. **[Disabled by default]**

Additionally, this script also chains the following scripts:
1. [BIOS Getter](https://github.com/theypsilon/MiSTer_BIOS_SCRIPTS) download bios for your installed systems from https://archive.org/
2. [MAME and HBMAME Getter](https://github.com/atrac17/MiSTer_MAME_SCRIPTS) download roms from https://archive.org/
3. [Arcade Organizer](https://github.com/theypsilon/_arcade-organizer). Creates a folder structure under `_Arcade/_Organized` for easy navigation to all the MRA files.

Special thanks to [Locutus73](https://github.com/Locutus73) for all his great work on the older [__Updater script__](https://github.com/MiSTer-devel/Updater_script_MiSTer). That __Updater script__ is no longer used by __Update All__ by default.
Additionally, this script also runs the [Arcade Organizer](https://github.com/theypsilon/_arcade-organizer). Which creates a folder structure under `_Arcade/_Organized` for easy navigation to all the MRA files.


## Setup
Expand All @@ -30,37 +30,22 @@ Then turn on your __MiSTer__, go to the _Scripts_ menu and run this script from
It will take around 30 minutes the first time you run it, but later runs should take much less time.


## Further Configuration

## Alternative Setups

Other setups that might be useful for you:
- [JTBIN ZIP](https://github.com/theypsilon/Update_All_MiSTer/releases/latest/download/update_all_jtbin.zip): Users that would like to download @jotego betas by default, can use this setup instead. You may also enable this option from the `Settings Screen` in the "Jotego Updater" submenu.
- [usb0 ZIP](https://github.com/theypsilon/Update_All_MiSTer/releases/latest/download/update_all_usb0.zip): For people using USB as primary storage. You will be able to run this script from there and install all cores, roms, and MRAs on `/media/usb0`.
- [DB9/SNAC8 ZIP](https://github.com/theypsilon/Update_All_MiSTer/releases/latest/download/update_all_db9_snac8.zip): Enables [Extended Native Controller Compatibility](https://github.com/theypsilon/Update_All_MiSTer/wiki#extended-native-controller-compatibility) for Genesis and NeoGeo/Supergun controllers.
In case you would like to configure Update All so it downloads exactly what you need, you can do it through the Settings screen.

![settings screen](https://github.com/theypsilon/Update_All_MiSTer/raw/master/setups/menu-2-0.jpeg "Settings Screen")

NOTE: You should extract also all the INI files contained in these.
The Settings screen will show up if you press UP in your controller or keyboard during the countdown showing up right after starting `update_all.sh` in your MiSTer. Please, don't forget to select `SAVE` to keep all the changes you have done before leaving.



## MiSTer Offline Setup

Check [updater-pc](./updater-pc) to download all the MiSTer files from your PC.

Useful if you can't connect your MiSTer to internet.



## Further Configuration

In case you would like to configure Update All so it downloads exactly what you need, you can do it through the Settings screen.

![settings screen](https://github.com/theypsilon/Update_All_MiSTer/raw/master/setups/menu-1-4.jpeg "Settings Screen")

The Settings screen will show up if you press UP in your controller or keyboard during the countdown showing up right after starting `update_all.sh` in your MiSTer. Please, don't forget to select `SAVE` to keep all the changes you have done before leaving.



## Funding

Consider funding [Alexey Melnikov "Sorgelig"](https://www.patreon.com/FPGAMiSTer) for his invaluable work on the [MiSTer project](https://github.com/MiSTer-devel/Main_MiSTer/wiki).
Expand All @@ -74,6 +59,8 @@ Check also other core developers that you might want to support:
* [furrtek](https://www.patreon.com/furrtek)
* [Ace](https://ko-fi.com/ace9921)
* [Blackwine](https://www.patreon.com/blackwine)
* [atrac17](https://www.patreon.com/atrac17)
* [Darren](https://ko-fi.com/darreno)

And finally, if you would like to support my work, you may also subscribe to my patreon here:

Expand All @@ -83,27 +70,19 @@ And finally, if you would like to support my work, you may also subscribe to my

## Supporters+ shout-out!

Antonio Villena, atrac17, birdybro, Hard Rich and MiSTerFPGA.co.uk
atrac17, birdybro, Hard Rich, MiSTerFPGA.co.uk and Tony Escobar

Thank you so much for supporting this project on [Patreon](https://www.patreon.com/bePatron?u=37499475)! You'll show up here if you become a **Supporter+**!


Special thanks to [Locutus73](https://github.com/Locutus73) for all his great work on the older [__Updater script__](https://github.com/MiSTer-devel/Updater_script_MiSTer). That __Updater script__ is no longer used by __Update All__ but it was an amazing source of inspiration.

## License

Copyright © 2020-2021, [José Manuel Barroso Galindo](https://twitter.com/josembarroso).
Copyright © 2020-2022, [José Manuel Barroso Galindo](https://twitter.com/josembarroso).
Released under the [GPL v3 License](LICENSE).



## Warning

I take no responsibility for any data loss or any damage you may incur because of the usage of this script.

Please check the README.md of the scripts being called by Update All:

https://github.com/MiSTer-devel/Downloader_MiSTer/<br>
https://github.com/theypsilon/Names_TXT_Updater_MiSTer<br>
https://github.com/atrac17/MiSTer_MAME_SCRIPTS<br>
https://github.com/theypsilon/_arcade-organizer<br>
https://github.com/theypsilon/MiSTer_BIOS_SCRIPTS<br>
MiSTer Scripts are run with root access in a pretty powerful device that has internet access. Be careful and do proper research before running any script on your device.
14 changes: 7 additions & 7 deletions dont_download.sh
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ set_default_options
UPDATE_ALL_VERSION="1.4"
UPDATE_ALL_PC_UPDATER="${UPDATE_ALL_PC_UPDATER:-false}"
UPDATE_ALL_OS="${UPDATE_ALL_OS:-MiSTer_Linux}"
UPDATE_ALL_LAUNCHER_MD5="ac10fbada40e3e5f133bc0eee0dd53d5"
UPDATE_ALL_LAUNCHER_MD5="9c44e188d397cd2d3fd3130bc3456bff"
UPDATE_ALL_PATREON_KEY_PATH="/media/fat/Scripts/update_all.patreonkey"
UPDATE_ALL_PATREON_KEY_SIZE="16384"
UPDATE_ALL_PATREON_KEY_MD5Q0="00e9f6acaec74650ddd38a14334ebaef"
Expand Down Expand Up @@ -362,12 +362,12 @@ initialize() {
rm "${OLD_SCRIPT_PATH}" 2> /dev/null || true
cp "${MAYBE_NEW_LAUNCHER}" "${OLD_SCRIPT_PATH}" || true

#echo
#echo "Update All's launcher script has just been upgraded."
#echo "Please execute this again to run the new version."
#echo "This is a one-time only process."
#echo
#exit 0
echo
echo "Update All's launcher script has just been upgraded."
echo "Please execute this again to run the new version."
echo "This is a one-time only process."
echo
exit 0
fi
fi
fi
Expand Down
Binary file added setups/menu-2-0.jpeg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
33 changes: 33 additions & 0 deletions src/__main__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env python3
# Copyright (c) 2022 José Manuel Barroso Galindo <[email protected]>

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

# You can download the latest version of this tool from:
# https://github.com/theypsilon/Update_All_MiSTer

import os
from update_all.main import main
from update_all.constants import DEFAULT_CURL_SSL_OPTIONS, \
DEFAULT_COMMIT, KENV_CURL_SSL, KENV_COMMIT, KENV_LOCATION_STR, DEFAULT_LOCATION_STR, KENV_DEBUG, DEFAULT_DEBUG

if __name__ == '__main__':
exit_code = main({
KENV_CURL_SSL: os.getenv(KENV_CURL_SSL, DEFAULT_CURL_SSL_OPTIONS),
KENV_COMMIT: os.getenv(KENV_COMMIT, DEFAULT_COMMIT),
KENV_LOCATION_STR: os.getenv(KENV_LOCATION_STR, DEFAULT_LOCATION_STR),
KENV_DEBUG: os.getenv(KENV_DEBUG, DEFAULT_DEBUG)
})

exit(exit_code)
20 changes: 20 additions & 0 deletions src/automated_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
# Copyright (c) 2022 José Manuel Barroso Galindo <[email protected]>

set -euo pipefail

echo "Type checks:"
python3 -m mypy src
echo

cd src
echo "Unit Tests:"
python3 -m unittest discover -s test/unit
echo
echo "Integration Tests:"
python3 -m unittest discover -s test/integration
echo
echo "SystemTests:"
python3 -m unittest discover -s test/system
echo
echo "Done"
42 changes: 42 additions & 0 deletions src/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/usr/bin/env bash
# Copyright (c) 2022 José Manuel Barroso Galindo <[email protected]>

set -euo pipefail

TEMP_ZIP1="$(mktemp -u).zip"
TEMP_ZIP2="$(mktemp -u).zip"
BIN="/tmp/dont_download.zip"
UUDECODE_CMD=$({ [[ "${MISTER:-false}" == "false" ]] && [[ "$(uname -s)" == "Darwin" ]] ; } && echo "uudecode -p" || echo "uudecode -o -")
EXPORTS="export COMMIT=$(git rev-parse --short HEAD)"

if [[ "${DEBUG:-false}" == "true" ]] ; then
EXPORTS="${EXPORTS}"$'\n'"export DEBUG=true"
fi

pin_metadata() {
touch -a -m -t 202108231405 "${1}"
}

cd src

find update_all -type f -iname "*.py" -print0 | while IFS= read -r -d '' file ; do pin_metadata "${file}" ; done
pin_metadata __main__.py
zip -q -0 -D -X -A -r "${TEMP_ZIP1}" __main__.py update_all -x "*/__pycache__/*"
pin_metadata "${TEMP_ZIP1}"
echo '#!/usr/bin/env python3' | cat - "${TEMP_ZIP1}" > "${TEMP_ZIP2}"
pin_metadata "${TEMP_ZIP2}"
rm "${TEMP_ZIP1}"
cd ..

cat <<-EOF
#!/usr/bin/env bash
set -euo pipefail
${EXPORTS}
${UUDECODE_CMD} "\${0}" | xzcat -d -c > "${BIN}"
chmod a+x "${BIN}"
"${BIN}" "\${1:-}"
exit 0
EOF

uuencode - < <(xzcat -z < "${TEMP_ZIP2}")
rm "${TEMP_ZIP2}" > /dev/null 2>&1 || true
Loading

0 comments on commit 9671fb4

Please sign in to comment.