wget

wget class ][ robots.txt 받아오기

wget.zip

1) subprocess 버전

# -*- coding: utf-8 -*-
# 
# write by kyoung chip , jang
#
# python 3.6
# pip list
#
import subprocess


class CWget :

    def __init__( self ) :
	
        pass

    def getFile( self, location , url ) :
	
        proc = subprocess.Popen(
		
            ['wget', '-O',location, url],
            stdout = subprocess.PIPE 
			
        )	
		
        out,err = proc.communicate()	
		
        print (  ( "output = %s \nerror = %s")  % ( out , err )   )

        


if __name__ == '__main__':

    w = CWget()
    w.getFile("c:\\k\python\\python\\robots.txt","https://www.google.com/robots.txt")

2) urllib3 버전1

# 
# write by kyoung chip , jang
#
# python 3.6
# pip list
#
import urllib.request

class CWget :

    def __init__( self ) :
	
        pass
		
    def getFile( self , location , url ) :
	
    
        urllib.request.urlretrieve( url , location )
		
		
if __name__ == '__main__':

    w = CWget()
    w.getFile( "c:\\k\\python\\python\\robots.txt" ,"https://www.google.com/robots.txt")

3) urllib3 버전2

# 
# write by kyoung chip , jang
#
# python 3.6
# pip list
#
import urllib.request
import shutil


class CWget :

    def __init__( self ) :
	
        pass
		
    def getFile( self , file_name , url ) :	
 
        with urllib.request.urlopen(url) as response, open(file_name, 'wb') as out_file:
            shutil.copyfileobj(response, out_file)
		
		
if __name__ == '__main__':

    w = CWget()
    w.getFile( "c:\\k\\python\\python\\robots.txt" ,"https://www.google.com/robots.txt")

4) urllib3 버전3

# -*- coding: utf-8 -*-
# 
# write by kyoung chip , jang
#
#
import urllib.request


class CWget :

    def __init__( self ) :
	
        pass
		
    def getFile( self , file_name , url ) :	
 
        with urllib.request.urlopen( url ) as response, open(file_name, 'wb') as out_file:
            data = response.read() 
            out_file.write(data)
		
		
if __name__ == '__main__':

    w = CWget()
    w.getFile( "c:\\k\\python\\python\\robots.txt" ,"https://www.google.com/robots.txt")

5) request 버전1

# -*- coding: utf-8 -*-
# 
# write by kyoung chip , jang
#
# pip install requests
#
import requests


class CWget :

    def __init__( self ) :
	
        pass
		
    def getFile( self , file_name , url ) :	
 
        with open( file_name , "wb" ) as file:

            response = requests.get( url )
            file.write( response.content )
		
		
if __name__ == '__main__':

    w = CWget()
    w.getFile( "c:\\k\\python\\python\\robots.txt" ,"https://www.google.com/robots.txt")