forked from dvehrs/podget
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathpodget
executable file
·3256 lines (2835 loc) · 153 KB
/
podget
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
# Useful to follow command execution and determine where an extra echo outputs during silent mode.
#set -x
# ----------------------------------------------------------------------------------------------------------------------------------
# Filename: podget {{{
# Maintainer: Dave Vehrs <davevehrs(at)users.sourceforge.net>
# Copyright: (c) 2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017 Dave Vehrs
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# Description: Podget is a simple bash script to automate the downloading and
# organizing of podcast content.
# Dependencies: bash, coreutils, debianutils, findutils, grep, gawk or mawk, libc-bin (for iconv), sed, and wget.
# Installation: cp podget.sh /usr/local/bin
# chmod 755 /usr/local/bin/podget.sh }}}
# ----------------------------------------------------------------------------------------------------------------------------------
# Exit Codes {{{
# "Reserved" Exit codes
# 1 General Error
# 2 Misuse of shell built-ins
# 126 Command invoked cannot execute
# 127 Command not found
# 128+n Invalid argument to exit
# 130 Script terminated by Control-C (128+2)
# 143 Script terminated by TERM signal (128+15)
# "Our" Exit codes
# Display Help (set to '0' because it is an valid exit condition, not an error.)
ERR_DISPLAYHELP=0
# Library directory not defined.
ERR_LIBNOTDEF=50
# Library directory available space below limit
ERR_LIBLOWSPACE=51
# Libc6 not installed. Cannot convert UTF16 feeds.
ERR_LIBC6NOTINSTALLED=60
# Another running session already exists.
ERR_RUNNINGSESSION=70
# OPML import error.
ERR_IMPORTOPML=80
# OPML export error.
ERR_EXPORTOPML=90
# }}}
# ----------------------------------------------------------------------------------------------------------------------------------
# Traps {{{
# FUNCNAME is declared with a default value in case the trap is triggered while
# outside a function.
trap 'EXIT_ERROR ${LINENO} ${?} ${FUNCNAME:-Unconfigured}' ERR
# trap to run CLEANUP function if program receives a TERM (kill) or INT (ctrl-c) signal
# - CLEANUP called in line for other normal exits.
trap 'CLEANUP_AND_EXIT 143' TERM
trap 'CLEANUP_AND_EXIT 130' INT
# }}}
# ----------------------------------------------------------------------------------------------------------------------------------
# Set Shell Options to catch errors ('man bash' for details) {{{
set -o errexit
set -o nounset
set -o pipefail
# Enable errtrace so that the ERR trap is inherited by functions
# NOTE: Causes an error about line 1841 where we try to filter various items out of
# the category and name. Speculation is it is caused by our use of the expr command
# that commonly gives non-zero exit status for commands that actually exited OK.
#
# We can get around the issue by adding a '|| true' to the end of each expr command
# but that seems a hack. What about replacing expr?
#set -o errtrace
# Enabling inheritance of errexit by command substitution subshells, this was added in Bash 4.4
# NOTE: Test command is run within a subshell where pipefail is disabled.
if (set +o pipefail && shopt inherit_errexit 2>&1 | grep -q invalid) ; then
echo "Bash added the 'inherit_errexit' shell option in version 4.4, please upgrade."
exit 1
fi
shopt -s inherit_errexit
# Enable extended glob matches.
shopt -s extglob
# }}}
# ----------------------------------------------------------------------------------------------------------------------------------
# Help text and default file formats {{{
: << HELP_STEXT
-c --config <FILE> Name of configuration file to use.
--create-config <FILE> Exit immediately after creating configuration file.
-C --cleanup Skip downloading and only run cleanup loop.
--cleanup_simulate Skip downloading and simulate running
cleanup loop.
Display files to be deleted.
--cleanup_days <COUNT> Number of days to retain files. Anything
older will be removed.
-d --dir_config <DIRECTORY> Directory that configuration files are
stored in.
--dir_session <DIRECTORY> Directory that session files are stored in.
-f --force Force download of items from each feed even
if they have already been downloaded.
--import_opml <FILE or URL> Import servers from OPML file or
HTTP/FTP URL.
--export_opml <FILE> Export serverlist to OPML file.
--import_pcast <FILE or URL> Import servers from iTunes PCAST file or
HTTP/FTP URL.
-l --library <DIRECTORY> Directory to store downloaded files in.
-n --no-playlist Do not create M3U playlist of new items.
-p --playlist-asx In addition to the default M3U playlist,
create an ASX Playlist. M3U playlist must be
created to convert to ASX.
--playlist-per-podcast Create playlist of new items for each podcast feed.
-r --recent <COUNT> Download only the <count> newest items from
each feed.
--serverlist <LIST> Serverlist to use.
-s --silent Run silently (for cron jobs).
--verbosity <LEVEL> Set verbosity level (0-4).
-v Set verbosity to level 1.
-vv Set verbosity to level 2.
-vvv Set verbosity to level 3.
-vvvv Set verbosity to level 4.
-h --help Display help.
HELP_STEXT
# }}}
# ----------------------------------------------------------------------------------------------------------------------------------
# Defaults {{{
#########################################################################################################################
## Do not configure here. Run podget once to install default user configuration files ($HOME/.podget) and edit there. ##
#########################################################################################################################
# Set DIR_LIBRARY, DIR_SESSION, and DIR_LOG in config file.
DIR_CONFIG="${HOME}/.podget"
CONFIG_CORE="podgetrc"
CONFIG_SERVERLIST="serverlist"
# DEFAULT FILENAME_BADCHARS used to test the configuration filenames and then unset. The value used later in the script may be
# set in the configuration file. If you have a genuine need to use one of these characters in the filenames of the serverlist or
# core configuration file then you may need to remove it from this definition. For all other uses, modify the setting in your
# configuration file (by default in podgetrc).
#
# This is a subset of the FILENAME_BADCHARS set in the default configuration file. Many of the symbols that have been removed are
# because they will cause other errors when used on the command line that prevent these checks from working.
FILENAME_BADCHARS="~#^=+{}[]:\"'?\\"
# Default VERBOSITY
# 0 == silent
# 1 == Warning messages only.
# 2 == Progress and Warning messages.
# 3 == Debug, Progress and Warning messages.
# 4 == All messages and wget set to maximum VERBOSITY.
VERBOSITY=2
# Auto-Cleanup.
# 0 == disabled
# 1 == delete any old content
CLEANUP=0
# Skip downloading and just run cleanup
# 0 == disable
CLEANUP_ONLY=0
# Simulate cleanup
CLEANUP_SIMULATE=0
# Number of days to keep files. Cleanup will remove anything
# older than this.
CLEANUP_DAYS=7
# Most Recent
# 0 == download all new items.
# 1+ == download only the <count> most recent
MOST_RECENT=0
# Force
# 0 == Only download new material.
# 1 == Force download all items even those you've downloaded before.
FORCE=0
# Install session. This gets called when script is first installed.
INSTALL_SESSION=0
# Fix filenames for FAT32 compatibility
MODIFY_FILENAME=0
# Stop downloads if available space drops below
MIN_SPACE=10000
# Date format for new playlist names
DATE_FORMAT=+%F
# ASX Playlists for Windows Media Player
# 0 == do not create
# 1 == create
ASX_PLAYLIST=0
# Enable playlist creation
NO_PLAYLIST=0
# Default DEBUG Disabled (Deletion of temporary files allowed), configured
# to allow its setting to be overridden from the command line.
DEBUG=${DEBUG:-0}
# Default DEBUG string leader
DEBUG_LEADER="DEBUG --"
# Order that items appear in feed.
# Default: DESCENDING - Newest items appear first, oldest last.
FEED_SORT_ORDER="DESCENDING"
# Create or update full playlist for each feed of all available items.
# 0 == Enable
# 1 == Disable
FEED_FULL_PLAYLIST=0
#########################################################################################################################
## Do not configure here. Run podget once to install default user configuration files ($HOME/.podget) and edit there. ##
#########################################################################################################################
# }}}
# ----------------------------------------------------------------------------------------------------------------------------------
# Text for default configuration files: {{{
: << TEXT_DEFAULT_CONFIG
# ----------------------------------------------------------------------------------------------------------------------------------
# Podget configuration file created by version @VERSION@
# [ NOTE: Do not delete version line as it will be used by future versions to
# to test if configuration files have been updated with any required changes.
# ----------------------------------------------------------------------------------------------------------------------------------
# File name and location configuration:
# Name of Server List configuration file
CONFIG_SERVERLIST=@SERVERLIST@
# Directory to store session files
# If this option is not configured then by default podget will place the session files in the directory defined by TMPDIR/podget or
# if it is not defined in the users shell then the session files will be placed in the directory /tmp/podget.
# If you prefer a different location, then configure this variable.
# DIR_SESSION=@HOME@/tmp/podget
# Directory where to store downloaded files
DIR_LIBRARY=@HOME@/POD
# Directory to store logs in
# By default, logs are stored in DIR_LIBRARY/.LOG
# If you prefer a different location, then configure this variable.
# DIR_LOG=@HOME@/POD/LOG
# Set logging file names
LOG_FAIL=errors
LOG_COMPLETE=done
# ----------------------------------------------------------------------------------------------------------------------------------
# Download Options:
# Wget base options
# Commonly used options:
# -c Continue interupted downloads - While this flag is commonly used there are feeds that it can
# cause "403 Forbidden" errors.
# -nH No host directories (overrides .wgetrc defaults if necessary)
# --proxy=off To disable proxy set by environmental variable http_proxy
# --no-check-certificate To disable HTTPS certificate checks. Useful for sites that may be using self-signed cerficates
# and not those from a trusted service authority.
# --prefer-family=IPv4/IPv6 When DNS provides a choice of addresses to connect to a host, attempt to connect to the specified
# address family first. If all addresses of the given family fail then the other family will be
# tried. If set to 'none' then the addresses will be tried in the order provided by the server
# regardless of which family they are in (this is effectively the default option).
#
# If you wish to force the use of IPv4 addresses only then you can use the "-4" or "--inet4-only"
# options. Conversely, if you want to force the use of IPv6 addresses then you can set the "-6"
# or "--inet6-only" options.
# --content-disposition [EXPERIMENTAL FEATURE] Wget will look for and use "Content-Disposition" headers received from the
# server. This can result in extra round-trips to the server for a "HEAD" request. This option
# is useful for servers that use the "Content-Disposition" header to hold the filename of the
# downloaded file rather than appending it to the URL. This has the potential to make some of
# Podget's FILENAME_FORMATFIX options unneeded.
#
# WARNING: Enabling this flag disables any download progress information from being passed on to
# the user. To debug errors that may occur during sessions with this flag enabled, it may be
# necessary to enable DEBUG and then examine the temporary files that are not deleted in
# DIR_SESSION.
#
# NOTE: This can be enable globally for all feeds here or if you want to enable it for only a few
# specific feeds, you can add "OPT_CONTENT_DISPOSITION" to their line in your serverlist.
#
# Wget options that include spaces need to be surrounded in quotes.
#
# WGET_BASEOPTS="-c --proxy=off --no-check-certificate"
# WGET_BASEOPTS="-nH --proxy=off --content-disposition"
# WGET_BASEOPTS="-c --prefer-family=IPv4"
# WGET_BASEOPTS="-c --prefer-family=IPv6"
WGET_BASEOPTS="-c -nH"
# Most Recent
# 0 == download all new items.
# 1+ == download only the <count> most recent
MOST_RECENT=0
# Force
# 0 == Only download new material.
# 1 == Force download all items even those you've downloaded before.
FORCE=0
# Autocleanup of old playlists and the files they list.
# 0 == disabled
# 1 == delete any old content
CLEANUP=0
# Number of days to keep files. Cleanup will remove anything
# older than this.
CLEANUP_DAYS=7
# Stop downloading if available space on the partition drops below value (in KB)
# default: 614400 (600MB)
MIN_SPACE=614400
# ----------------------------------------------------------------------------------------------------------------------------------
# Playlist Options:
# Disable playlist creation [ No need to comment out other playlist variables ]
# 0 == create
# 1 == do not create
NO_PLAYLIST=0
# Build playlists (comment out or set to a blank string to accept default format: New-).
PLAYLIST_NAMEBASE=New-
# Date format for new playlist names
# +%F = YYYY-MM-DD like 2014-01-15 (DEFAULT)
# +%m-%d-%Y = MM-DD-YYYY like 01-15-2014
# For other options 'man date'
#
# Date options that include spaces need to be surrounded in quotes.
#
DATE_FORMAT=+%F
# ASX Playlists for Windows Media Player
# 0 == do not create
# 1 == create
ASX_PLAYLIST=0
# ----------------------------------------------------------------------------------------------------------------------------------
# Filename Suffix:
# Add suffix to the filename of every file downloaded to allow for subsequent scripts to detect the newly downloaded files and work
# on them. Examples of this would be scripts to run id3v2 to force a standard genre for all MP3 files downloaded or to use mp3gain
# to normalize files to have the same volume.
#
# A period (.) will automatically be added between the filename and tag like so:
# filename.mp3.newtag
#
# Tags will not be added to filenames as they are added to the playlists. It will be necessary for any script that you run to
# process the files remove the tag for the playlists to work.
#
# If this variable is undefined or commented out, then by default no suffix will be added.
# FILENAME_SUFFIX="newtag"
# ----------------------------------------------------------------------------------------------------------------------------------
# Downloaded Filename Cleanup Options:
#
# These options are for the filenames downloaded from the feeds. We will try to clean then up rather than interrupting the script
# execution.
# Filename Cleanup: For FAT32 filename compatability (Feature Request #1378956)
# Tested with the following characters: !@#$%^&*()_-+=||{[}]:;"'<,>.?/
#
# The \`, \" and \\ characters need to be escaped with a leading backslash.
#
# Bad Character definitions need to be surrounded in quotes.
#
# NOTE: FILENAME_BADCHARS is also used to test for characters that commonly cause errors in directory names. This can cause
# FILENAME_BADCHARS to be reported as part of an error for configuration issues with DIR_SESSION, DIR_LOG, DIR_LIBRARY and podcast
# FEED_NAME and FEED_CATEGORY.
FILENAME_BADCHARS="\`~!#$^&=+{}*[]:;\"'<>?|\\"
# Filename Replace Character: Character to use to replace any/all
# bad characters found.
FILENAME_REPLACECHAR=_
# When you run podget at a VERBOSITY of 3 or 4, it may appear that the filename format fixes are done out of order. That is because
# they are named as they are created and as new fixes have been developed, those with more detailed exclusionary conditions have had
# to be done before those with more generic conditions. Looking for improvements to fix this issue.
# Filename Cleanup 2: Some RSS Feeds (like the BBC World News Bulletin)
# download files with names like filename.mp3?1234567. Enable this mode
# to fix the format to filename1234567.mp3.
# 0 == disabled
# 1 == enabled (default)
FILENAME_FORMATFIX=1
# Filename Cleanup 3: Filenames of feeds hosted by LBC Plus corrupted.
# Fixed per MoonUnit's feature request (#1660764)
#
# Takes an URL that looks like: http://lbc.audioagain.com/shared/audio/stream.mp3?guid=2007-03/14<...snip>
# <snip...>a7766e8ad2748269fd347eaee2b2e3f8&source=podcast.php&channel_id=88
#
# Which normally creates a file named: a7766e8ad2748269fd347eaee2b2e3f8&source=podcast.php&channel_id=88
#
# This fix extracts the date of the episode and changes the filename to 2007-03-14.mp3
# 0 == disabled
# 1 == enabled (default)
FILENAME_FORMATFIX2=1
# Filename Cleanup 4: Filenames of feeds hosted by CatRadio.cat need fixing.
# Fixed per Oriol Rius's Bug Report (#1744705)
#
# Downloaded filenames look like: 1189153569775.mp3?programa=El+mat%ED+de+Catalunya+R%E0dio&podcast=y
# This fix removes everything after the .mp3
#
# NOTE: Testing in 2017 reveals changes in CatRadio's URL format that hampers this fix.
#
# Downloaded filenames now look like: 1487257264030.mp3&programa=Bon+dia%2C+malparits%21&var10=Neix+%2BCatR%E0dio%2C+el+dial+digital+de+Catalunya+R%E0dio&var11=video&var15=1783&var20=Podcast&var29=Audio&var3=951439&var14=951439&v25=Catalunya+R%E0dio&var19=16/02/17&var12=Tall&var18=45194
#
# Two changes cause issues:
# 1. Change of '?' to '&' for designating options.
# 2. Use of forward slashes in date (var19) mess up some of our other filename extraction.
#
# However the fix for these podcasts is now simpler. If in our serverlist, we use either
# the OPT_FILENAME_LOCATION or OPT_CONTENT_DISPOSTION option for these feedlists then the
# filename will be correctly extracted. This leaves us with a long number as the filename,
# however if we also enable the OPT_FILENAME_RENAME_MDATE option then the filename is prefixed
# with the last modification date of the file which helps list the files in an order that
# makes sense.
#
# 0 == disabled (default)
# 1 == enabled
FILENAME_FORMATFIX3=0
# Filename Cleanup 5: When the filename is part of the URL and the actual filename stays the same for
# all items listed.
#
# Download URLs look like: http://feeds.theonion.com/~r/theonion/radionews/~5/213589629/podcast_redirect.mp3
# Where 213589629 is the unique filename.
#
# This filename change is disabled by default because it may cause unintended changes to the filename.
#
# 0 == disabled (default)
# 1 == enabled
FILENAME_FORMATFIX4=0
# Filename Cleanup 6: Remove "?referrer=rss" from the end of filenames as included in some feeds like
# those from Vimcasts.org. Setup to work for MP3, M4V, OGG and OGV files.
#
# Feed URLs: http://vimcasts.org/feeds/ogg
# http://vimcasts.org/feeds/quicktime
#
# In the feed, enclosure URLs look like: http://media.vimcasts.org/videos/1/show_invisibles.ogv?referrer=rss
#
# 0 == disabled
# 1 == enabled (default)
FILENAME_FORMATFIX5=1
# Filename Cleanup 7: Removes the trailing part of the filename after the '?'.
# Fixed at the request of Joerg Schiermeier
#
# For dealing with enclosures like those formatted in the ZDF podcast.
# Feed URL: http://www.zdf.de/ZDFmediathek/podcast/1193018?view=podcast
# Example enclosure:
# http://podfiles.zdf.de/podcast/zdf_podcasts/101103_backstage_afo_p.mp4?2010-11-03+06-42
#
# 0 == disabled
# 1 == enabled (default)
FILENAME_FORMATFIX6=1
# Filename Cleanup 8:
# This fix is for feeds that assign the same filename to be downloaded for each
# enclosure and then embedded the actual filename of the object to be saved in
# the media_url= parameter. This fix extracts that name and uses it for the
# saved file.
#
# 0 == disabled
# 1 == enabled (default)
FILENAME_FORMATFIX7=1
# Filename Cleanup 9:
# This fix is for feeds like Smodcast. It removes the "?client_id=<string>"
# from the end of each enclosure url in the feed.
#
# NOTE: To fully fix the filenames on feeds like Smodcast, this fix should
# be used in conjunction with FILENAME_FORMATFIX4.
#
# Example URL: http://api.soundcloud.com/tracks/62837276/stream.mp3?client_id=a427c512429c9c90e58de7955257879c
# Fixed filename: 62837276_stream.mp3
#
# 0 == disabled
# 1 == enabled (default)
FILENAME_FORMATFIX8=1
# Filename Cleanup 10:
#
# This is a fix for podcast feeds formatted like those for Audioboo. Removes everything after the ?
# in the filename. Attempted to make this fix generic enough to work with a variety of feeds of mp3, mp4,
# ogg and ogv files.
#
# Feed URL: http://audioboo.fm/users/39903/boos.rss
# Example URL: http://audioboo.fm/boos/1273271-mw-123-es-wird-fruhling.mp3?keyed=true&source=rss
# Fixed Filename: 1273271-mw-123-es-wird-fruhling.mp3
#
# NOTE: On Aug 30 2018, this fix was updated to also fix feeds formated like those from viertausendhertz.de.
#
# Feed URL: http://viertausendhertz.de/feed/podcast/systemfehler
# Example URL: https://viertausendhertz.de/podcast-download/1538/sf04.mp3?v=1470947681&source=feed
# Fixed Filename: sf04.mp3
#
# 0 == disabled
# 1 == enabled (default)
FILENAME_FORMATFIX9=1
# Filename Cleanup 11:
#
# This is an attempt to fix feeds hosted on Apple ITunes. The enclosure URL from these feeds defines the
# the filename as a long string of numbers and letter. It's not very descriptive. However, after the
# filename and a '?', in the information passed down to the application as part of the URL, we can
# extract the episode name for each podcast. It is that name that this fix will use for the filename,
# with a few character replacements to insure good filenames.
#
# 0 == disabled
# 1 == enabled (default)
FILENAME_FORMATFIX10=1
# ----------------------------------------------------------------------------------------------------------------------------------
# DEBUG
#
# Enabling debug will:
# 1. Stop podget from automatically deleting some temporary files in DIR_SESSION.
# 2. Enable additional messages to track progress.
#
# 0 == disabled (default)
# 1 == enabled
# ${DEBUG:-0} == Sets DEBUG to disabled if it is not already set. This allows the user to enabled it
# from the command line with "DEBUG=1 podget"
#
#DEBUG=${DEBUG:-0}
# ----------------------------------------------------------------------------------------------------------------------------------
TEXT_DEFAULT_CONFIG
: << TEXT_DEFAULT_SERVERLIST
# Default Server List for podget
#
# Default format with category and name:
# <url> <category> <name>
#
# Alternate Formats:
# 1. With a category but no name.
# <url> <category>
# 2. With a name but no category (2 ways).
# <url> No_Category <name>
# <url> . <name>
# 3. With neither a category or name.
# <url>
#
# For additional formating documentation, please refer to 'man podget'.
#
#FEEDS:
# ----------------------------------------------------------------------------------------------------------------------------------
http://thelinuxlink.net/tllts/tllts.rss LINUX The Linux Link
TEXT_DEFAULT_SERVERLIST
: << TEXT_ASX_BEGINNING
<ASX version = "3.0">
<PARAM NAME = "Encoding" VALUE = "UTF-8" />
<PARAM NAME = "Custom Playlist Version" VALUE = "V1.0 WMP8 for CE" />
TEXT_ASX_BEGINNING
: << TEXT_ASX_END
</ASX>
TEXT_ASX_END
# }}}
# ----------------------------------------------------------------------------------------------------------------------------------
# Functions {{{
# Function: ARRAY_CONTAINS
# Test if Array contains element.
# Returns 0 if found, or 1 if not found.
# ARGUMENTS:
# ${1} == String to search for
# ${2} == Array to search within (also called ${@:2})
ARRAY_CONTAINS() {
local element
for element in "${@:2}"; do [[ "${element}" == "${1}" ]] && return 0; done
return 1
}
# Function: CLEANUP_AND_EXIT
# Closes session and removes lock file (if it exists)
# ARGUMENTS:
# ${1} == Exit Status to report.
CLEANUP_AND_EXIT() {
local EXITSTATUS=${1}
if (( VERBOSITY >= 2 )) ; then
#echo -en "\nClosing session"
printf '\n%s' "Closing session"
fi
if [[ -n ${DIR_SESSION+set} && -f ${DIR_SESSION}/podget.$$ ]]; then
if (( DEBUG == 0 )); then
if (( VERBOSITY >= 2 )) ; then
printf '%s' " and removing lock file"
fi
if (( VERBOSITY >= 4 )) ; then
echo
rm -fv "${DIR_SESSION}"/podget.$$
else
rm -f "${DIR_SESSION}"/podget.$$
fi
else
printf '\n%s\n' "${DEBUG_LEADER} Not deleting ${DIR_SESSION}/podget.$$"
fi
fi
if (( (VERBOSITY >= 2) && (VERBOSITY <= 3) )); then
printf '%s\n' "."
elif (( (VERBOSITY == 1) || (VERBOSITY > 3) )); then
echo
fi
exit "${EXITSTATUS}"
}
display_shelp() {
echo; echo "Usage $0 [options]"; echo
sed --silent -e '/HELP_STEXT$/,/^HELP_STEXT/p' "$0" | sed -e '/HELP_STEXT/d'
}
# Function: DIRECTORY_CHECK 'Name of Variable to be tested'
# Simple function to verify that unsafe characters are not used in directory names.
DIRECTORY_CHECK() {
# Variables have a default value of 'UNCONFigured' because this word and combination
# of capitalization is unlikely to be used. This allows us to catch improperly
# formated calls to DIRECTORY_CHECK.
#
# Uses variable indirection, The '!' introduces indirection which can be read
# to say "Get the value of the variable named this".
local TEST_STRING=${!1:-"UNCONFigured"}
# The second use simply reports the name of the variable to be tested.
local TEST_VARIABLE=${1:-"UNCONFigured"}
local TEST_FAIL=0
local OFFENDING_CHARS=""
if [[ ${TEST_STRING} == "UNCONFigured" ]]; then
echo "Improperly formated call to DIRECTORY_CHECK."
return 1
fi
# Test if FILENAME_BADCHARS is configured, if it is then check filenames to
# prevent the use of disallowed characters.
if [[ -n ${FILENAME_BADCHARS+set} ]]; then
for (( i=0; i<${#FILENAME_BADCHARS}; i++ )); do
local TEST_CHAR=${FILENAME_BADCHARS:$i:1}
# Grep looks for --fixed-strings so that certain characters are not
# interpreted as regular expressions (like ^ $ or /)
if grep --quiet --fixed-strings "${TEST_CHAR}" <<<"${TEST_STRING}"; then
OFFENDING_CHARS="${OFFENDING_CHARS}${TEST_CHAR}"
# This test must come first because it will set TEST_FAIL to 1 regardless
# of how many offending characters are found. Given that this test can be
# reported multiple times, we do not want it to cause additional suggestions
# to be given to the user below.
TEST_FAIL=1
fi
done
fi
# consult Shellcheck SC2076 for why I choose this construct rather than using regex (=~) checks
if [[ ${TEST_STRING} = *"../"* ]]; then
TEST_FAIL=$((TEST_FAIL+2))
fi
if [[ ${TEST_STRING} = *"*"* ]]; then
TEST_FAIL=$((TEST_FAIL+4))
fi
# This test will create duplicate suggestions as the back-slash character also appears in
# the default FILENAME_BADCHARS.
if [[ ${TEST_STRING} = *"\\000"* ]]; then
TEST_FAIL=$((TEST_FAIL+8))
fi
if (( TEST_FAIL != 0 )); then
echo "DIRECTORY CHECK ERROR: ${TEST_VARIABLE} = ${TEST_STRING}"
echo
echo "Suggestion(s):"
local COUNT=0
while (( TEST_FAIL != 0 )); do
if (( (8<=TEST_FAIL) && (TEST_FAIL<=150) )); then
COUNT=$((COUNT+1))
echo " ${COUNT}. '\\000' cannot be used in directory names as mkdir expects"
echo " to get a null terminated string and '\\000' is considered 'end of string'."
if (( TEST_FAIL >= 8 )); then
TEST_FAIL=$((TEST_FAIL-8))
fi
elif (( (4<=TEST_FAIL) && (TEST_FAIL<=7) )); then
COUNT=$((COUNT+1))
echo " ${COUNT}. The asterisk should not be used in directories as they are commonly used"
echo " to designate a wild card for expansion in Bash variables."
if (( TEST_FAIL >= 4 )); then
TEST_FAIL=$((TEST_FAIL-4))
fi
elif (( (2<=TEST_FAIL) && (TEST_FAIL<=3) )); then
COUNT=$((COUNT+1))
echo " ${COUNT}. Directories should not contain two periods and a slash in conjunction."
echo " If you need to save certain files outside of the Podcast Library directory,"
echo " defined by this podgetrc, the proper solution is to create a second podgetrc"
echo " with the new library location defined and to run podget with the --config"
echo " command line option to designate the podgetrc file to use."
if (( TEST_FAIL >= 2 )); then
TEST_FAIL=$((TEST_FAIL-2))
fi
elif ((1==TEST_FAIL)); then
COUNT=$((COUNT+1))
echo " ${COUNT}. Attempts to use characters disallowed by FILENAME_BADCHARS."
echo " Either remove the offending characters from the configured directory"
echo " or FILENAME_BADCHARS."
echo " Configured characters not allowed: ${FILENAME_BADCHARS}"
echo " Offending character(s): ${OFFENDING_CHARS}"
if (( TEST_FAIL >= 1 )); then
TEST_FAIL=$((TEST_FAIL-1))
fi
fi
done
if [[ ${TEST_VARIABLE} == "FEED_NAME" || ${TEST_VARIABLE} == "FEED_CATEGORY" ]]; then
return 1
else
CLEANUP_AND_EXIT 1
fi
fi
}
EXIT_ERROR() {
# Name of script
local JOB_NAME
JOB_NAME=$(basename "$0")
# The following three variables are configured with a default value in case
# the function is called without options set.
local LINENUM="${1:-"Unconfigured"}" # Line with error
local EXITSTATUS="${2:-"Unconfigured"}" # exit status of error
local FUNCTION="${3:-"Unconfigured"}" # If error occurred in a function, its name will be listed.
printf '\n%s\n %-15s %s\n' "Error:" "Script:" "${JOB_NAME}"
# Function line only appears if it has been set to value other than the
# default. Works on the assumption that "Unconfigured" is not likely to be
# chosen as a function name.
if [[ ${FUNCTION} != "Unconfigured" ]]; then
printf ' %-15s %s\n' "Function:" "${FUNCTION}"
fi
printf ' %-15s %s\n %-15s %s\n' "At line:" "${LINENUM}" "Exit Status:" "${EXITSTATUS}"
printf '\n%s\n' "Context:"
# Test is awk installed, if so use it. If not, then use tools from coreutils.
if hash awk 2>/dev/null; then
# This line works and adds a ">>>" to designate the offending line but adds
# awk as a script dependency.
awk 'NR>L-4 && NR<L+4 { printf "%-5d%3s%s\n",NR,(NR==L?">>>":""),$0 }' L="${LINENUM}" "${0}"
else
# This line works and only depends on coreutils
pr -tn "${0}" | tail -n+$((LINENUM - 3)) | head -n7
fi
CLEANUP_AND_EXIT 1
}
# Function: FILENAME_CHECK 'Name of Variable to be tested'
# This function tests the filenames used by podget locally for various configuration and log files. While these checks have
# some similarity to those applied to downloaded files the major difference is that violations of these rules will interrupt
# the execution of podget and podget will attempt to fix the other filenames but many not always succeed.
# Arguments:
FILENAME_CHECK() {
# Variables have a default value of 'UNCONFigured' because this word and combination
# of capitalization is unlikely to be used. This allows us to catch improperly
# formated calls to FILENAME_CHECK.
#
# Uses variable indirection, The '!' introduces indirection which can be read
# to say "Get the value of the variable named this".
local TEST_STRING=${!1:-"UNCONFigured"}
# The second use simply reports the name of the variable to be tested.
local TEST_VARIABLE=${1:-"UNCONFigured"}
local TEST_FAIL=0
local OFFENDING_CHARS=""
if [[ ${TEST_STRING} == "UNCONFigured" ]]; then
echo "Improperly formated call to FILENAME_CHECK."
return 1
fi
# Test if FILENAME_BADCHARS is configured, if it is then check filenames to
# prevent the use of disallowed characters.
if [[ -n ${FILENAME_BADCHARS+set} ]]; then
for (( i=0; i<${#FILENAME_BADCHARS}; i++ )); do
local TEST_CHAR=${FILENAME_BADCHARS:$i:1}
# Grep looks for --fixed-strings so that certain characters are not
# interpreted as regular expressions (like ^ $ or /)
if grep --quiet --fixed-strings "${TEST_CHAR}" <<<"${TEST_STRING}"; then
OFFENDING_CHARS="${OFFENDING_CHARS}${TEST_CHAR}"
# This test must come first because it will set TEST_FAIL to 1 regardless
# of how many offending characters are found. Given that this test can be
# reported multiple times, we do not want it to cause additional suggestions
# to be given to the user below.
TEST_FAIL=1
fi
done
fi
if [[ -z ${TEST_STRING##*/*} ]]; then
# First test remove PATH from TEST_FILENAME variable.
local TEST_DIRECTORY="${TEST_STRING%/*}"
local TEST_FILENAME="${TEST_STRING##*/}"
if [[ -n "${TEST_DIRECTORY}" ]]; then
TEST_FAIL=$((TEST_FAIL+2))
fi
# Remove directory from string to be tested for following tests.
TEST_STRING=${TEST_FILENAME}
fi
# Configuration files should not be hidden by leading periods and trailing periods can cause issues on some file systems or
# operating systems. Test if filename begins or ends with a period (.)
if [[ ${TEST_STRING:0:1} == "." || ${TEST_STRING:(-1):1} == "." ]]; then
TEST_FAIL=$((TEST_FAIL+4))
fi
if (( TEST_FAIL != 0 )); then
echo
case "${TEST_VARIABLE}" in
"CONFIG_CORE" )
echo "Configuration filename specified by -c or --create-config violates the following rules..."
;;
"CMDL_SERVERLIST" )
echo "Serverlist filename specified by --serverlist violates the following rules..."
;;
"CONFIG_SERVERLIST" )
echo "Default Serverlist filename violates the following rules..."
;;
"LOG_FAIL" )
echo "LOG_FAIL defined in ${CONFIG_CORE} violates the following rules..."
;;
"LOG_COMPLETE" )
echo "LOG_COMPLETE defined in ${CONFIG_CORE} violates the following rules..."
;;
* )
echo "${TEST_VARIABLE} violates the following rules..."
;;
esac
echo
echo "Suggestion(s):"
COUNT=0
while (( TEST_FAIL != 0 )); do
case ${TEST_FAIL} in
# Included as an example of how other errors could be added with
# a binary progression for the values they add to TEST_FAIL.
[4-7])
COUNT=$((COUNT+1))
echo " ${COUNT}. Remove leading or trailing period from ${TEST_STRING}"
if (( TEST_FAIL > 1 )); then
TEST_FAIL=$((TEST_FAIL-4))
fi
;;
[2-3])
COUNT=$((COUNT+1))
echo " ${COUNT}. Filenames should not include any directory configuration."
echo " Remove the directory configuration."
case "${TEST_VARIABLE}" in
"CONFIG_CORE" | "CMDL_SERVERLIST" | "CONFIG_SERVERLIST" )
echo " If you need to specify a directory other than the default,"
echo " use the -d or --dir_config command line options."
;;
"LOG_FAIL" | "LOG_COMPLETE" )
echo " If you wish to specify another location to store the logs,"
echo " then configure the DIR_LOG variable in your ${CONFIG_CORE}"
;;
esac
if (( TEST_FAIL >= 2 )); then
TEST_FAIL=$((TEST_FAIL-2))
fi
;;
1)
COUNT=$((COUNT+1))
echo " ${COUNT}. Attempts to use characters disallowed by FILENAME_BADCHARS."
echo " Either remove the offending characters from the configured directory"
echo " or FILENAME_BADCHARS."
echo " Configured characters not allowed: ${FILENAME_BADCHARS}"
echo " Offending character(s): ${OFFENDING_CHARS}"
if (( TEST_FAIL >= 1 )); then
TEST_FAIL=$((TEST_FAIL-1))
fi
;;
esac
done
CLEANUP_AND_EXIT 1
fi
}
# Function: filenameFixFormat ${1} ${2}
# Arguments:
# ${1} == name of variable to hold return string
# ${2} == string to fix the format of
filenameFixFormat() {
# variable to hold returned value.
local VAR_RETURN=${1}
# Filename to be modified.
# Set original value for filename format fixes and character substitutions.
# Set according to what is passed as the second argument to function.
local MODIFIED_FILENAME=${2}
if [[ -n ${FILENAME_FORMATFIX+set} || -n ${FILENAME_FORMATFIX2+set} || -n ${FILENAME_FORMATFIX3+set} ||
-n ${FILENAME_FORMATFIX4+set} || -n ${FILENAME_FORMATFIX5+set} || -n ${FILENAME_FORMATFIX6+set} ||
-n ${FILENAME_FORMATFIX7+set} || -n ${FILENAME_FORMATFIX8+set} || -n ${FILENAME_FORMATFIX9+set} ||
-n ${FILENAME_FORMATFIX10+set} ]]; then
if (( VERBOSITY >= 3 )) ; then
printf '%-30s %s\n' "ORIGINAL FILENAME:" "${MODIFIED_FILENAME}"
fi
fi
# Note: Filename format fixes that have more specific conditions come first. More generic last. This is
# because a fix with too liberal a condition can prevent a more specific fix from running. Fixes are named in
# the order they were created, so it may appear that they are out of order. By changing the order that they are
# executed in, it is possible to have more enabled by default.
#
# TODO: Create exclusionary conditions for the fixes that are out of order to restore sanity to this list.
#
# FILENAME_FORMATFIX has been moved to the end of the order.
#
# FILENAME_FORMATFIX4 is not part of this function and is called immediately after this function ends.
# Filename format fix for podcasts hosted on http://lbc.audioagain.com.
if [[ -n ${FILENAME_FORMATFIX2+set} ]] && (( FILENAME_FORMATFIX2 > 0 )); then
if (( $(expr "${MODIFIED_FILENAME}" : "[0-9a-zA-Z]\\+[&]amp;source=podcast.php[&]amp;channel_id=[0-9]\\+\$") > 0 )); then
MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed 's/.*stream.mp3[?]guid=\([0-9]\+\)-\([0-9]\+\)\/\([0-9]\+\)\/.*/\1-\2-\3.mp3/')
if (( DEBUG == 1 )) ; then
echo "${DEBUG_LEADER} FILENAME FORMAT(2) FIXED: ${MODIFIED_FILENAME}"
fi
fi
fi
# Filename format fix for podcasts hosted on http://www.catradio.cat
if [[ -n ${FILENAME_FORMATFIX3+set} ]] && (( FILENAME_FORMATFIX3 > 0 )); then
if (( $(expr "${MODIFIED_FILENAME}" : "[0-9]\\+\\.mp3\\?[&]programa=[0-9a-Z+=%&;]*\$") > 0 )); then
# shellcheck disable=SC2001
MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed 's/\(.*\)\.mp3\(.*\)/\1\.mp3/g')
if (( DEBUG == 1 )) ; then
echo "${DEBUG_LEADER} FILENAME FORMAT(3) FIXED: ${MODIFIED_FILENAME}"
fi
fi
fi
# Remove "?referrer=rss" from filename as included with some feeds like Vimcasts.org
if [[ -n ${FILENAME_FORMATFIX5+set} ]] && (( FILENAME_FORMATFIX5 > 0 )); then
if (( $(expr "${MODIFIED_FILENAME}" : "[-0-9a-zA-Z_]\\+\\.[gmopv34]\\+[?]referrer=rss") > 0 )); then
MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed -r 's/([-A-Za-z0-9_]+.[ogmpv34]+)[?]referrer=rss/\1/g')
if (( DEBUG == 1 )) ; then
echo "${DEBUG_LEADER} FILENAME FORMAT(5) FIXED: ${MODIFIED_FILENAME}"
fi
fi
fi
# ZDF podcast filename fix
if [[ -n ${FILENAME_FORMATFIX6+set} ]] && (( FILENAME_FORMATFIX6 > 0 )); then
if (( $(expr "${MODIFIED_FILENAME}" : "[-_0-9a-zA-Z]\\+\\.[gmopv34]\\+[?][-_+0-9]\\+") > 0 )); then
MODIFIED_FILENAME=$(echo "${MODIFIED_FILENAME}" | sed -ru 's/([-_A-Za-z0-9]+.[ogmpv3-4]+)[?][-+0-9]*/\1/g')
if (( DEBUG == 1 )) ; then
echo "${DEBUG_LEADER} FILENAME FORMAT(6) FIXED: ${MODIFIED_FILENAME}"
fi
fi
fi
# media_url cleanup
# This fix was inspired by the Radio France podcast feed. Each enclosure URL in the feed had the same filename