-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fixed issue #1, and some other improvements
- Loading branch information
1 parent
9cb3a74
commit 2b85231
Showing
4 changed files
with
207 additions
and
88 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,5 +15,6 @@ | |
| **/test.py | ||
|
|
||
| *.list | ||
| **/*.html | ||
|
|
||
| download/ | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| import os, re | ||
| import os.path | ||
| import glob | ||
| from pathlib import Path | ||
| import codecs | ||
| import shutil | ||
|
|
||
| def check_file_exists(FILE): | ||
| '''Check if the FILE exists''' | ||
| return Path(FILE).is_file() | ||
|
|
||
|
|
||
| def check_dir_exists(DIR): | ||
| '''Check if the DIR exists''' | ||
| return Path(DIR).is_dir() | ||
|
|
||
|
|
||
| def get_basename(FILE): | ||
| ''' | ||
| Return the basename of a file. e.g. example.txt -> example | ||
| ''' | ||
| return os.path.splitext(os.path.basename(FILE))[0] | ||
|
|
||
|
|
||
| def file_path(FILE): | ||
| return os.path.dirname(os.path.realpath(FILE)) + os.sep | ||
|
|
||
|
|
||
| def script_path(): | ||
| return os.path.dirname(os.path.realpath(__file__)) | ||
|
|
||
|
|
||
| def line_prepender(filename, line): | ||
| ''' | ||
| Add line to the head of a file | ||
| ''' | ||
| with open(filename, 'r+') as f: | ||
| content = f.read() | ||
| f.seek(0, 0) | ||
| f.write(line.rstrip('\r\n') + '\n' + content) | ||
|
|
||
|
|
||
| def get_file_list(folder): | ||
| file_list = [] | ||
| for path, subdirs, files in os.walk(folder): | ||
| for name in files: | ||
| file_list.append(os.path.join(path, name)) | ||
| return file_list | ||
|
|
||
|
|
||
| def purge_folder(folder, filePattern='*'): | ||
| # filelist = [ f for f in os.listdir(folder) ] #if f.endswith(".bak") ] | ||
| filelist = glob.glob(folder + os.sep + filePattern) | ||
| for f in filelist: | ||
| # print(f) | ||
| os.remove(os.path.join(f)) # using glob | ||
| # os.remove(os.path.join(folder, f)) # using listdir | ||
|
|
||
|
|
||
| def create_folder(folderName): | ||
| '''Create folder if not exists''' | ||
| my_file = Path(folderName) | ||
| if not my_file.is_dir(): | ||
| print('Folder {} not found, creating a new one'.format(folderName)) | ||
| os.mkdir(folderName) | ||
|
|
||
|
|
||
| def text_replace_in_file(pattern, string, file): | ||
| '''Replace pattern with string in file''' | ||
| with open(file) as f: | ||
| replaced_script = re.sub(pattern, string, f.read(), flags=re.IGNORECASE) | ||
| with open(file, 'w') as f: | ||
| f.write(replaced_script) | ||
|
|
||
|
|
||
| def convert_encode2utf8(sourceFileName, targetFileName, srcEncoding = 'utf-16'): | ||
| BLOCKSIZE = 1048576 # or some other, desired size in bytes | ||
| with codecs.open(sourceFileName, 'r', 'utf-16') as sourceFile: | ||
| with codecs.open(targetFileName, 'w', 'utf-8') as targetFile: | ||
| while True: | ||
| contents = sourceFile.read(BLOCKSIZE) | ||
| if not contents: | ||
| break | ||
| targetFile.write(contents) | ||
|
|
||
|
|
||
| def remove_junk_line(FILE, junkwords): | ||
| ''' | ||
| Remove the line that contains junkwords | ||
| ''' | ||
| with open(FILE) as oldfile, open(FILE + 'tmp', 'w') as newfile: | ||
| for line in oldfile: | ||
| if not junkwords in line: | ||
| newfile.write(line) | ||
| shutil.move(FILE + 'tmp', FILE) | ||
|
|
||
| if __name__ == '__main__': | ||
| print(get_file_list('E:\\')) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,62 +1,64 @@ | ||
| import requests | ||
| import os | ||
|
|
||
| import toolkit_file | ||
|
|
||
| VIDEO_LIST = 'youtube_video_download.list' | ||
| proxy = '127.0.0.1:1080' | ||
| downloadPath = 'download/' | ||
|
|
||
| toolkit_file.create_folder(downloadPath) | ||
|
|
||
| def remove_illegal_char(fileName): | ||
| ''' | ||
| Remove reserved characters from file name | ||
| ''' | ||
| RESERVED_CHAR = ['<', '>', ':', '"', '/', '\\', '|', '?', '*',] | ||
| ''' | ||
| Remove reserved characters from file name | ||
| ''' | ||
| RESERVED_CHAR = ['<', '>', ':', '"', '/', '\\', '|', '?', '*',] | ||
|
|
||
| for char in RESERVED_CHAR: | ||
| fileName = fileName.replace(char, '_') | ||
| return fileName | ||
| for char in RESERVED_CHAR: | ||
| fileName = fileName.replace(char, '_') | ||
| return fileName | ||
|
|
||
| def read_list(): | ||
| with open(VIDEO_LIST, 'r') as f: | ||
| downloadList_tmp = f.read().split('\n') | ||
| with open(VIDEO_LIST, 'r') as f: | ||
| downloadList_tmp = f.read().split('\n') | ||
|
|
||
| downloadList = [] | ||
| for i in downloadList_tmp: | ||
| # get fileName, downloadUrl | ||
| if not i: | ||
| continue | ||
| downloadList = [] | ||
| for i in downloadList_tmp: | ||
| # get fileName, downloadUrl | ||
| if not i: | ||
| continue | ||
|
|
||
| downloadItem = i.split(' |#| ') | ||
| downloadItem[0] = remove_illegal_char(downloadItem[0].strip()) + '.mp4' | ||
| downloadList.append(downloadItem) | ||
| downloadItem = i.split(' |#| ') | ||
| downloadItem[0] = remove_illegal_char(downloadItem[0].strip()) + '.mp4' | ||
| downloadList.append(downloadItem) | ||
|
|
||
| return(downloadList) | ||
| return(downloadList) | ||
|
|
||
| def download_file(fileName, url): | ||
| ''' | ||
| Download file with proxy | ||
| ''' | ||
| print('Downloading {}'.format(fileName)) | ||
| proxies = {'http': 'http://{}'.format(proxy), | ||
| 'https': 'https://{}'.format(proxy)} | ||
| res = requests.get(url,proxies=proxies) | ||
| con = res.content | ||
| with open(fileName, 'wb') as f: | ||
| f.write(con) | ||
| print('Downloading finished') | ||
| ''' | ||
| Download file with proxy | ||
| ''' | ||
| print('Downloading {}'.format(fileName)) | ||
| proxies = {'http': 'http://{}'.format(proxy), | ||
| 'https': 'https://{}'.format(proxy)} | ||
| res = requests.get(url,proxies=proxies) | ||
| con = res.content | ||
| with open(fileName, 'wb') as f: | ||
| f.write(con) | ||
| print('Downloading finished') | ||
|
|
||
| def create_download_dir(): | ||
| script_dir = os.path.dirname(os.path.realpath(__file__)) | ||
| try: | ||
| if downloadPath not in os.listdir(script_dir): | ||
| os.mkdir(downloadPath) | ||
| except FileExistsError as e: | ||
| pass | ||
| script_dir = os.path.dirname(os.path.realpath(__file__)) | ||
| try: | ||
| if downloadPath not in os.listdir(script_dir): | ||
| os.mkdir(downloadPath) | ||
| except FileExistsError as e: | ||
| pass | ||
|
|
||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| create_download_dir() | ||
| for i in read_list(): | ||
| download_file(downloadPath + i[0], i[1]) | ||
| create_download_dir() | ||
| for i in read_list(): | ||
| download_file(downloadPath + i[0], i[1]) |