387 lines
16 KiB
Python
387 lines
16 KiB
Python
|
import requests
|
|||
|
|
from bs4 import BeautifulSoup
|
|||
|
|
import re
|
|||
|
|
import json
|
|||
|
|
import asyncio
|
|||
|
|
import aiohttp
|
|||
|
|
import time
|
|||
|
|
from googleapiclient.discovery import build
|
|||
|
|
from google.oauth2 import service_account
|
|||
|
|
import pandas as pd
|
|||
|
|
import openpyxl
|
|||
|
|
|
|||
|
|
|
|||
|
|
SCOPES = [
|
|||
|
|
'https://www.googleapis.com/auth/spreadsheets',
|
|||
|
|
'https://www.googleapis.com/auth/drive'
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
credentials = service_account.Credentials.from_service_account_file('creds_service_acc.json', scopes=SCOPES)
|
|||
|
|
|
|||
|
|
# spreadsheet_service = build('sheets', 'v4', credentials=credentials)
|
|||
|
|
# drive_service = build('drive', 'v3', credentials=credentials)
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
spreadsheet_service = build('sheets', 'v4', credentials=credentials)
|
|||
|
|
except:
|
|||
|
|
DISCOVERY_SERVICE_URL = 'https://sheets.googleapis.com/$discovery/rest?version=v4'
|
|||
|
|
spreadsheet_service = build('sheets', 'v4', credentials=credentials, discoveryServiceUrl=DISCOVERY_SERVICE_URL)
|
|||
|
|
|
|||
|
|
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36'}
|
|||
|
|
|
|||
|
|
def files_jso():
|
|||
|
|
with open("./tovar.json", "w", encoding='utf8') as file:
|
|||
|
|
file.write(f'[')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def files_js():
|
|||
|
|
|
|||
|
|
with open("./tovar.json", "r", encoding='utf8') as file:
|
|||
|
|
x = file.read()
|
|||
|
|
|
|||
|
|
with open("./tovar.json", "w", encoding='utf8') as file1:
|
|||
|
|
file1.write(x[:-2])
|
|||
|
|
|
|||
|
|
with open("./tovar.json", "a", encoding='utf8') as file3:
|
|||
|
|
file3.write(f']')
|
|||
|
|
|
|||
|
|
def files_tovar():
|
|||
|
|
with open("./urltovar.txt", "w", encoding='utf8') as file:
|
|||
|
|
file.write(f'')
|
|||
|
|
with open("./catalogall.txt", "w", encoding='utf8') as file:
|
|||
|
|
file.write(f'')
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
def prov_pagen():
|
|||
|
|
file1 = open('./catalog.txt', "r")
|
|||
|
|
while True: # Цикл для блокнота что бы пробегал по каждой строке)
|
|||
|
|
line = file1.readline()
|
|||
|
|
if not line:
|
|||
|
|
break
|
|||
|
|
url = line.strip()
|
|||
|
|
s = requests.Session()
|
|||
|
|
r = s.get(url, headers=headers).text
|
|||
|
|
soup = BeautifulSoup(r, 'html.parser')
|
|||
|
|
pagen = soup.find_all('select', {"id" : 'nb_page_items'})
|
|||
|
|
p = str(pagen)
|
|||
|
|
u = ( ' '.join(re.findall(r'>([^<>]+)</option> </select>', str(p))) )
|
|||
|
|
# print(u)
|
|||
|
|
if len(u) == 0:
|
|||
|
|
print(url)
|
|||
|
|
with open("./catalogall.txt", "a") as file:
|
|||
|
|
file.write(url + '\n')
|
|||
|
|
else:
|
|||
|
|
print(f'{url}?n={u}')
|
|||
|
|
with open("./catalogall.txt", "a") as file:
|
|||
|
|
file.write(f'{url}?n={u}' + '\n')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def catalog():
|
|||
|
|
url='https://allsuvenir.com.ua/ru/'
|
|||
|
|
s = requests.Session()
|
|||
|
|
res = s.get(url, headers=headers).text
|
|||
|
|
|
|||
|
|
|
|||
|
|
soup = BeautifulSoup(res, 'html.parser')
|
|||
|
|
items = soup.find_all('div', class_= 'list')
|
|||
|
|
# print(items)
|
|||
|
|
url = ( ' '.join(re.findall(r'<a href="([^<>]+)" title="">', str(items))) )
|
|||
|
|
line = url.replace(' ', '\n')
|
|||
|
|
# catalog = re.findall(r'"\s*([^"]*?)\s*"', line)#json.loads(line)
|
|||
|
|
# print(len(catalog))
|
|||
|
|
with open("./catalog.txt", "w") as file:
|
|||
|
|
file.write(line)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def urls_tovar():
|
|||
|
|
file1 = open('./catalogall.txt', "r")
|
|||
|
|
while True: # Цикл для блокнота что бы пробегал по каждой строке)
|
|||
|
|
line = file1.readline()
|
|||
|
|
if not line:
|
|||
|
|
break
|
|||
|
|
url = line.strip()
|
|||
|
|
# url = url1+'&n=135'
|
|||
|
|
print(url)
|
|||
|
|
# url = 'https://allsuvenir.com.ua/ru/114-zazhigalki?id_category=114&n=135'
|
|||
|
|
s = requests.Session()
|
|||
|
|
r = s.get(url, headers=headers).text
|
|||
|
|
|
|||
|
|
soup = BeautifulSoup(r, 'html.parser')
|
|||
|
|
items = soup.find_all('ul', class_='product_list grid row')
|
|||
|
|
# print(items)
|
|||
|
|
|
|||
|
|
url = ( ' '.join(re.findall(r'"product-name" href="([^<>]+)" itemprop="url"', str(items))) )
|
|||
|
|
line = url.replace(' ', '\n')
|
|||
|
|
# tovar = re.findall(r'"\s*([^"]*?)\s*"', line)#json.loads(line)
|
|||
|
|
print(line)
|
|||
|
|
with open("./urltovar.txt", "a") as file:
|
|||
|
|
file.write(line + '\n')
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def main1():
|
|||
|
|
async with aiohttp.ClientSession() as session:
|
|||
|
|
|
|||
|
|
file1 = open('./urltovar.txt', "r")
|
|||
|
|
while True: # Цикл для блокнота что бы пробегал по каждой строке)
|
|||
|
|
line = file1.readline()
|
|||
|
|
if not line:
|
|||
|
|
|
|||
|
|
break
|
|||
|
|
url = line.strip()
|
|||
|
|
# url = 'https://allsuvenir.com.ua/ru/glavnaya/1344-mk14.html?search_query=MK14&results=2'
|
|||
|
|
|
|||
|
|
# print(url)
|
|||
|
|
async with session.get(url, headers=headers)as response:
|
|||
|
|
# await asyncio.sleep(0.2)
|
|||
|
|
|
|||
|
|
|
|||
|
|
r = await aiohttp.StreamReader.read(response.content)
|
|||
|
|
soup = BeautifulSoup(r, 'html.parser')
|
|||
|
|
print(f'===========> {url}')
|
|||
|
|
try:
|
|||
|
|
table = soup.find_all('div', {"id" : 'center_column'})
|
|||
|
|
for item in table:
|
|||
|
|
json_list = []
|
|||
|
|
# js = []
|
|||
|
|
try:
|
|||
|
|
name = item.find('div', class_='pb-center-column col-xs-12 col-sm-4').find('h1').get_text(strip=True)
|
|||
|
|
print(name)
|
|||
|
|
code = item.find('p', {"id" : 'product_reference'}).get_text(strip=True)
|
|||
|
|
codes = code.replace('Артикул', '')
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
ops = item.find('div', {"id" : 'short_description_content'}).get_text()
|
|||
|
|
print(ops)
|
|||
|
|
except Exception:
|
|||
|
|
ops = ""
|
|||
|
|
|
|||
|
|
|
|||
|
|
prices = item.find('span', {"id" : 'our_price_display'}).text
|
|||
|
|
price = prices[:prices.find('$')]
|
|||
|
|
fotos = item.find('ul', {"id" : 'thumbs_list_frame'})
|
|||
|
|
foto1 = ( ' '.join(re.findall(r'"other-views" href="([^<>]+)" title="', str(fotos))) )
|
|||
|
|
foto = foto1.replace(' ', ', ')
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
js1 = {'code': f'ALL_{codes}',
|
|||
|
|
'name': name,
|
|||
|
|
'price': price,
|
|||
|
|
'foto': foto,
|
|||
|
|
'op': ops}
|
|||
|
|
|
|||
|
|
|
|||
|
|
with open("./tovar.json", "a", encoding='utf8') as file:
|
|||
|
|
|
|||
|
|
json.dump(js1, file, ensure_ascii=False)
|
|||
|
|
file.write(',' + '\n')
|
|||
|
|
with open("./urltovar.txt", "r", encoding='utf8') as f:
|
|||
|
|
lines = f.readlines()
|
|||
|
|
|
|||
|
|
pattern = re.compile(re.escape(url))
|
|||
|
|
with open("./urltovar.txt", "w", encoding='utf8') as f:
|
|||
|
|
for line in lines:
|
|||
|
|
result = pattern.search(line)
|
|||
|
|
if result is None:
|
|||
|
|
f.write(line)
|
|||
|
|
|
|||
|
|
print(codes)
|
|||
|
|
except Exception as a:
|
|||
|
|
# else:
|
|||
|
|
print(a)
|
|||
|
|
except Exception as a:
|
|||
|
|
# else:
|
|||
|
|
print(a)
|
|||
|
|
time.sleep(100)
|
|||
|
|
print('____________________________________________________________')
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
def search_id():
|
|||
|
|
js = []
|
|||
|
|
with open("./tovar.json", "r", encoding='utf8') as file:#,encoding='utf8') as file:
|
|||
|
|
rr = json.load(file)
|
|||
|
|
for r in rr:
|
|||
|
|
codes = r['code']
|
|||
|
|
names = r['name']
|
|||
|
|
price = float(r['price'])
|
|||
|
|
foto = r['foto']
|
|||
|
|
op = r['op']
|
|||
|
|
# print(op)
|
|||
|
|
|
|||
|
|
ange_name = 'Export Products Sheet!H1:H400'
|
|||
|
|
spreadsheet_id = '1KQUjDeeOE8fIC1DlA7N8VL32Ee4Wh0CgC28p3cggsuc' # КОД страницы)
|
|||
|
|
range_ = 'Export Products Sheet!AF1:AF15000'
|
|||
|
|
# value_render_option = '40'
|
|||
|
|
date_time_render_option = ''
|
|||
|
|
request = spreadsheet_service.spreadsheets().values().get(spreadsheetId=spreadsheet_id, range=range_)#, valueRenderOption=value_render_option)#, dateTimeRenderOption=date_time_render_option)
|
|||
|
|
response = request.execute()
|
|||
|
|
|
|||
|
|
# print(type(response))
|
|||
|
|
a = (response['values'])
|
|||
|
|
search_ids = [f'{codes}']
|
|||
|
|
# print(search_ids)
|
|||
|
|
time.sleep(1)
|
|||
|
|
if search_ids in a:
|
|||
|
|
|
|||
|
|
# b = (a.index( search_ids) )
|
|||
|
|
# print('go search')
|
|||
|
|
|
|||
|
|
for i, j in enumerate(a):
|
|||
|
|
if j == search_ids:
|
|||
|
|
# print(i+1)
|
|||
|
|
stroka = i+1
|
|||
|
|
print(f'[=>] Нашёл в таблице! Код товара: {codes} В стороке: {stroka} Цена: {price}')
|
|||
|
|
if price > 0.00:
|
|||
|
|
tovar = [['+']]
|
|||
|
|
price1 = [[f'{price}']]
|
|||
|
|
resp = spreadsheet_service.spreadsheets().values().update(
|
|||
|
|
spreadsheetId=spreadsheet_id,
|
|||
|
|
range="Export Products Sheet!P{}".format(stroka),
|
|||
|
|
valueInputOption="RAW",
|
|||
|
|
body={'values' : tovar }).execute()
|
|||
|
|
resp2 = spreadsheet_service.spreadsheets().values().update(
|
|||
|
|
spreadsheetId=spreadsheet_id,
|
|||
|
|
range="Export Products Sheet!Z{}".format(stroka),
|
|||
|
|
valueInputOption="RAW",
|
|||
|
|
body={'values' : price1 }).execute()
|
|||
|
|
|
|||
|
|
elif price == 0.00:
|
|||
|
|
tovar = [['-']]
|
|||
|
|
price1 = [[f'{price}']]
|
|||
|
|
resp = spreadsheet_service.spreadsheets().values().update(
|
|||
|
|
spreadsheetId=spreadsheet_id,
|
|||
|
|
range="Export Products Sheet!P{}".format(stroka),
|
|||
|
|
valueInputOption="RAW",
|
|||
|
|
body={'values' : tovar }).execute()
|
|||
|
|
resp2 = spreadsheet_service.spreadsheets().values().update(
|
|||
|
|
spreadsheetId=spreadsheet_id,
|
|||
|
|
range="Export Products Sheet!Z{}".format(stroka),
|
|||
|
|
valueInputOption="RAW",
|
|||
|
|
body={'values' : price1 }).execute()
|
|||
|
|
# print ('-------!!!!!!!')
|
|||
|
|
else:
|
|||
|
|
print('not')
|
|||
|
|
|
|||
|
|
|
|||
|
|
else:
|
|||
|
|
#
|
|||
|
|
# pass
|
|||
|
|
js1 = {'code': codes,
|
|||
|
|
'name': names,
|
|||
|
|
'price': price,
|
|||
|
|
'foto': foto,
|
|||
|
|
'op': op}
|
|||
|
|
js.append(js1)
|
|||
|
|
# print(js)
|
|||
|
|
DF = pd.DataFrame.from_dict(js)
|
|||
|
|
DF.to_excel( './all_notfiles.xlsx', sheet_name='no product', index=False)
|
|||
|
|
# print(f'[=>] Записал в файл {codes}')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def find_elements_starting_with(a, prefix):
|
|||
|
|
return [s for s in a if s.startswith(prefix)]
|
|||
|
|
|
|||
|
|
def finds_ids():
|
|||
|
|
# with open("./idsexes.txt", "w", encoding='utf8') as file:
|
|||
|
|
# file.write(f'')
|
|||
|
|
|
|||
|
|
|
|||
|
|
lis = []
|
|||
|
|
valuesss = [[int(0)]]
|
|||
|
|
with open("./tovar.json", "r", encoding='utf8') as file:#,encoding='utf8') as file:
|
|||
|
|
rr = json.load(file)
|
|||
|
|
for r in rr:
|
|||
|
|
codes1 = r['code']
|
|||
|
|
codes = f'ALL_{codes1}'
|
|||
|
|
lis.append(codes)
|
|||
|
|
# print(lis)
|
|||
|
|
ange_name = 'Export Products Sheet!H1:H400'
|
|||
|
|
spreadsheet_id = '1KQUjDeeOE8fIC1DlA7N8VL32Ee4Wh0CgC28p3cggsuc'#'1c5gAmgzLqbNfAC6a4DL7F4KDPXI5xn3joD0Z8Kxr-jY' # КОД страницы)
|
|||
|
|
range_ = 'Export Products Sheet!AF2:AF15000'
|
|||
|
|
# value_render_option = '40'
|
|||
|
|
date_time_render_option = ''
|
|||
|
|
request = spreadsheet_service.spreadsheets().values().get(spreadsheetId=spreadsheet_id, range=range_)#, valueRenderOption=value_render_option)#, dateTimeRenderOption=date_time_render_option)
|
|||
|
|
response = request.execute()
|
|||
|
|
|
|||
|
|
# print(type(response))
|
|||
|
|
a = (response['values'])
|
|||
|
|
time.sleep(2)
|
|||
|
|
st = str(a).replace('[', '')
|
|||
|
|
st1 = st.replace(']', '')
|
|||
|
|
st2 = st1.replace("'", '')
|
|||
|
|
|
|||
|
|
li = list(st2.split(", "))
|
|||
|
|
prefix = 'ALL'
|
|||
|
|
resultgoog = find_elements_starting_with(li, prefix)
|
|||
|
|
result = []
|
|||
|
|
# print(a)
|
|||
|
|
for iii in resultgoog:
|
|||
|
|
if iii not in lis:
|
|||
|
|
result.append(iii)
|
|||
|
|
search_ids = [f'{iii}']
|
|||
|
|
time.sleep(2)
|
|||
|
|
if search_ids in a:
|
|||
|
|
|
|||
|
|
# b = (a.index( search_ids) )
|
|||
|
|
# print('go search')
|
|||
|
|
|
|||
|
|
for i, j in enumerate(a):
|
|||
|
|
# time.sleep(1)
|
|||
|
|
if j == search_ids:
|
|||
|
|
# print(i+1)
|
|||
|
|
stroka = i+2
|
|||
|
|
print(f'[=>] Нашёл в таблице! Код товара: {search_ids} В стороке: {stroka}! Этого товара не было при парсинге!')
|
|||
|
|
tovar = [['-']]
|
|||
|
|
resp = spreadsheet_service.spreadsheets().values().update(
|
|||
|
|
spreadsheetId=spreadsheet_id,
|
|||
|
|
range="Export Products Sheet!P{}".format(stroka),
|
|||
|
|
valueInputOption="RAW",
|
|||
|
|
body={'values' : tovar }).execute()
|
|||
|
|
|
|||
|
|
# resp = spreadsheet_service.spreadsheets().values().update(
|
|||
|
|
# spreadsheetId=spreadsheet_id,
|
|||
|
|
# range="Export Products Sheet!Z{}".format(stroka),
|
|||
|
|
# valueInputOption="RAW",
|
|||
|
|
# body={'values' : valuesss }).execute()
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
files_tovar()
|
|||
|
|
catalog()
|
|||
|
|
prov_pagen()
|
|||
|
|
urls_tovar()
|
|||
|
|
files_jso()
|
|||
|
|
print('[*] Проверяю что осталось в файле')
|
|||
|
|
time.sleep(10)
|
|||
|
|
check_finish1 = True
|
|||
|
|
while check_finish1:
|
|||
|
|
a = len(re.findall(r"[\n]+?", open('./urltovar.txt').read()))
|
|||
|
|
print(a)
|
|||
|
|
if a > 1:
|
|||
|
|
try:
|
|||
|
|
loop = asyncio.get_event_loop()
|
|||
|
|
loop.run_until_complete(main1())
|
|||
|
|
check_finish1 = False
|
|||
|
|
except Exception:
|
|||
|
|
print('[*]Новое подключние')
|
|||
|
|
time.sleep(10)
|
|||
|
|
else:
|
|||
|
|
break
|
|||
|
|
# loop = asyncio.get_event_loop()
|
|||
|
|
# loop.run_until_complete(main1())
|
|||
|
|
files_js()
|
|||
|
|
# try:
|
|||
|
|
# search_id()
|
|||
|
|
# except Exception as e:
|
|||
|
|
# print(e)
|
|||
|
|
# print('[*] Не залил весь товар.')
|
|||
|
|
# #
|
|||
|
|
# try:
|
|||
|
|
# finds_ids()
|
|||
|
|
# except Exception as e:
|
|||
|
|
# print(e)
|
|||
|
|
# print('[*] Не проверил весь товар которого не было на сайте.')
|
|||
|
|
|