This commit is contained in:
Xander 2025-01-25 16:45:18 +02:00
parent e9ade1feda
commit bbf0c949dc
9 changed files with 415 additions and 185 deletions

4
.gitignore vendored
View File

@ -24,7 +24,10 @@ node_modules/
# Игнорируем файлы виртуального окружения Python
.venv/
.env/
# Игнорируем папки для тестов и alembic
test/
alembic/
alembic.ini
# Игнорируем скомпилированные файлы
*.out
@ -38,6 +41,7 @@ test/
/build/
/target/
# Игнорируем файлы базы данных
*.sqlite
*.db

View File

@ -3,12 +3,13 @@
- [x] Профиль аккаунта по ссылке
- [x] Добавил модельку для работы
- [ ] Вакансия по ссылке (частично)
- [ ] Ежедневный обмен вакансий (Ожидаю доступы)
> [!NOTE]
> Обмен по времени, один раз в сутки!
> Обмен по времени, один раз в сутки! Проверка объявлений до 10 дней в таблице Jobs столбик days_posted!
> Пишет логи!

View File

@ -1,2 +1,3 @@
USERNAME=
PASSWD=
DATABASE_URL=

202
main.py
View File

@ -1,195 +1,33 @@
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel, HttpUrl
from urllib.parse import urlparse, parse_qs
from linkedin_api import Linkedin
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
import logging
from utils.logging_setup import configure_global_logging
from datetime import datetime
from dotenv import load_dotenv
import os
load_dotenv()
# # Ваши учетные данные LinkedIn
from routers import profile, jobs
configure_global_logging()
# Инициализация приложения
app = FastAPI(
title="linkedin API",
description="API с linkedin получаем данные по URL",
version="1.0.0"
)
version="1.0.0")
username = os.getenv('USERNAME')
password = os.getenv('PASSWD')
api = Linkedin(username, password)
class Profile(BaseModel):
name: str
title: str
url: HttpUrl
class UrlModel(BaseModel):
link_profile: HttpUrl # Используем HttpUrl для проверки валидности URL
@app.post("/link_profile")
async def profile_url(data: UrlModel):
# Проверяем, что ссылка начинается с "https://xander"
if not str(data.link_profile).startswith("https://www.linkedin.com/"):
# Возвращаем исключение с кодом 400 и сообщением об ошибке
raise HTTPException(status_code=400, detail="Неправильная ссылка")
url_profiles = str(data.link_profile)
path = urlparse(url_profiles).path
print(path)
# Пропускаем '/in/' и получаем последнюю часть
profiles = path.split("/")[2]
print(profiles)
profile = api.get_profile(profiles)
logging.info(f"{profile}")
firstName = profile['firstName']
lastName = profile['lastName']
locationName = profile['locationName']
geoLocationName = profile['geoLocationName']
headline = profile['headline']
educations = profile['education']
education = [
{
"schoolName": item.get("schoolName"),
"Field_of_Study": item.get("fieldOfStudy"),
"Degree": item.get('degreeName'),
"Grade": "??",
"Start Date": item.get("timePeriod", {}).get("startDate", {}).get("year"),
"End Date": item.get("timePeriod", {}).get("endDate", {}).get("year"),
"Currently study here": '??',
"Description": "??"
}
for item in educations
]
experiences = profile['experience']
experience = [
{
"companyName": item.get("companyName"),
"title": item.get("title"),
"Employment type": "??",
"Location": item.get('geoLocationName'),
"Location Type": "??",
"Start Date": {
"month": item.get("timePeriod", {}).get("startDate", {}).get("month"),
"year": item.get("timePeriod", {}).get("startDate", {}).get("year"),
},
"End Date":{
"month": item.get("timePeriod", {}).get("endDate", {}).get("month"),
"year": item.get("timePeriod", {}).get("endDate", {}).get("year"),
},
"Currently study here": '??',
"Description": item.get('description')
}
for item in experiences
]
volunteers = profile['volunteer']
volunteer = [
{
"Organization": item.get("companyName"),
"Role": item.get("role"),
"Cause": item.get("cause"),
"Location": item.get('geoLocationName'),
"Start Date": {
"month": item.get("timePeriod", {}).get("startDate", {}).get("month"),
"year": item.get("timePeriod", {}).get("startDate", {}).get("year"),
},
"End Date":{
"month": item.get("timePeriod", {}).get("endDate", {}).get("month"),
"year": item.get("timePeriod", {}).get("endDate", {}).get("year"),
},
"Currently study here": '??',
"Description": item.get('description')
}
for item in volunteers
]
all_skills = profile['skills']
skills = [item['name'] for item in all_skills]
all_languages = profile['languages']
languages = [item['name'] for item in all_languages]
# Подключение роутеров
app.include_router(profile.router, tags=["Profile"])
app.include_router(jobs.router, tags=["Jobs"])
# Проверка состояния сервера
@app.get("/", tags=["Check"])
async def check():
return {"status": "ok"}
# Если профиль не в виде словаря, возвращаем его напрямую
return {"FirstName": firstName,
"LastName":lastName,
"Location": f"{geoLocationName} {locationName}",
"Statement":headline,
'Education - list': education,
'Experience - list': experience,
"Volunteering - list": volunteer,
"skills": skills,
"languages":languages}
@app.post("/link_vacancy")
async def vacancy_url(data: UrlModel):
if not str(data.link_profile).startswith("https://www.linkedin.com/"):
# Возвращаем исключение с кодом 400 и сообщением об ошибке
raise HTTPException(status_code=400, detail="Неправильная ссылка")
query_params = parse_qs(urlparse(str(data.link_profile)).query)
current_job_id = query_params.get("currentJobId", [None])[0]
jobs = api.get_job(current_job_id)
location = jobs['formattedLocation']
title = jobs['title']
listedAt = jobs['listedAt']
# Преобразование из миллисекунд в секунды и конвертация
future_date = datetime.fromtimestamp(listedAt / 1000)
# Текущая дата
current_date = datetime.now()
# Разница в днях
difference = abs((future_date - current_date).days)
# print(f"Разница в днях: {difference}")
jobPostingId = jobs['jobPostingId']
# workplaceTypesResolutionResults = jobs['workplaceTypesResolutionResults']
# Извлекаем все localizedName
# localized_names = [value['localizedName'] for value in workplaceTypesResolutionResults.values()]
# print(localized_names)
# localized_name = workplaceTypesResolutionResults['urn:li:fs_workplaceType:2']['localizedName']
link = f'https://www.linkedin.com/jobs/view/{current_job_id}/'
return {"job_id": current_job_id,
"job_title": title,
"minimum_annual_salary": f"minimum_annual_salary",
"salary_currency": 'salary_currency',
'location_type': "location_type",
'location': location,
"job_level": "job_level",
"job_type": 'job_type',
"days_posted": difference,
"hourly_rate": "hourly_rate",
"link": link}
# Глобальный middleware для обработки необработанных исключений
@app.middleware("http")
async def catch_exceptions_middleware(request: Request, call_next):
try:
return await call_next(request)
except Exception as e:
logging.error(f"Unhandled exception: {str(e)}")
return JSONResponse(content={"error": "Internal Server Error"}, status_code=500)

68
models/models.py Normal file
View File

@ -0,0 +1,68 @@
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import sessionmaker, declarative_base
from sqlalchemy import Column, Integer, String, Text, DateTime, Float, Boolean
from sqlalchemy.dialects.mysql import JSON
from sqlalchemy.sql import func
from dotenv import load_dotenv
import os
# Загрузка переменных окружения
load_dotenv()
DATABASE_URL = os.getenv('DATABASE_URL') # Убедитесь, что URL настроен на асинхронный движок
# Например: "mysql+aiomysql://user:password@host:port/database"
# Создание базы данных
Base = declarative_base()
engine = create_async_engine(DATABASE_URL, echo=True)
async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
# Profile model
class Profile(Base):
__tablename__ = "profile"
id = Column(Integer, primary_key=True, index=True)
link = Column(String(255), unique=True, nullable=True)
first_name = Column(String(255), nullable=False)
last_name = Column(String(255), nullable=False)
pronouns = Column(String(50), nullable=True)
email = Column(String(255), nullable=True)
telephone = Column(String(20), nullable=True)
location = Column(String(255), nullable=True)
statement = Column(Text, nullable=True)
skills = Column(JSON, nullable=True) # List of skills
websites = Column(JSON, nullable=True) # List of website links
languages = Column(JSON, nullable=True) # List of languages with proficiency
date = Column(DateTime, server_default=func.now())
# Jobs model
class Job(Base):
__tablename__ = "jobs"
job_id = Column(Integer, primary_key=True, index=True)
job_title = Column(String(255), nullable=False)
job_company = Column(String(255), nullable=False)
minimum_annual_salary = Column(Float, nullable=True)
salary_currency = Column(String(10), nullable=True)
location_type = Column(String(50), nullable=False) # Remote, On-site, Hybrid
location = Column(String(255), nullable=False)
job_level = Column(String(50), nullable=False) # Entry-level, Junior, Mid, etc.
job_type = Column(String(50), nullable=False) # Full-time, Part-time, etc.
days_posted = Column(Integer, nullable=False)
hourly_rate = Column(Float, nullable=True)
link = Column(String(2083), nullable=False) # URL for the job posting
link_company = Column(String(2083), nullable=False) # URL for the job posting
active = Column(Boolean, default=True) # Indicates if the job is active
# Create database tables
async def init_db():
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
# Example for dependency injection in FastAPI
async def get_session():
async with async_session() as session:
yield session

10
models/start.py Normal file
View File

@ -0,0 +1,10 @@
import asyncio
from models import init_db # Предполагается, что файл с моделями называется models.py
async def main():
await init_db()
print("Таблицы созданы!")
if __name__ == "__main__":
asyncio.run(main())

View File

@ -1,20 +1,29 @@
aiomysql==0.2.0
alembic==1.14.1
annotated-types==0.7.0
anyio==4.8.0
beautifulsoup4==4.12.3
certifi==2024.12.14
charset-normalizer==3.4.1
click==8.1.8
databases==0.9.0
fastapi==0.115.6
greenlet==3.1.1
h11==0.14.0
idna==3.10
linkedin-api
linkedin-api @ git+https://github.com/tomquirk/linkedin-api.git@dacec3c9f03b4f1fbddb4f7dfdef57ea408e40aa
lxml==5.3.0
Mako==1.3.8
MarkupSafe==3.0.2
pydantic==2.10.5
pydantic_core==2.27.2
PyMySQL==1.1.1
python-dotenv==1.0.1
requests==2.32.3
schedule==1.2.2
sniffio==1.3.1
soupsieve==2.6
SQLAlchemy==2.0.37
starlette==0.41.3
typing_extensions==4.12.2
urllib3==2.3.0

107
routers/jobs.py Normal file
View File

@ -0,0 +1,107 @@
from fastapi import FastAPI, APIRouter, Depends, Request, HTTPException, Form
from datetime import datetime
from dotenv import load_dotenv
import os
from fastapi.responses import HTMLResponse
from fastapi.responses import JSONResponse
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from pydantic import BaseModel, HttpUrl
from urllib.parse import urlparse, parse_qs
from linkedin_api import Linkedin
import logging
from utils.logging_setup import configure_global_logging
load_dotenv()
# # Ваши учетные данные LinkedIn
configure_global_logging()
router = APIRouter()
username = os.getenv('USERNAME')
password = os.getenv('PASSWD')
api = Linkedin(username, password)
DATABASE_URL = os.getenv('DATABASE_URL') # Ваши параметры подключения
engine = create_engine(DATABASE_URL)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
class UrlModel(BaseModel):
link_profile: HttpUrl # Используем HttpUrl для проверки валидности URL
async def get_session() -> AsyncSession:
async with async_session() as session:
yield session
@router.post("/link_vacancy")
async def vacancy_url(data: UrlModel):
if not str(data.link_profile).startswith("https://www.linkedin.com/"):
# Возвращаем исключение с кодом 400 и сообщением об ошибке
raise HTTPException(status_code=400, detail="Неправильная ссылка")
query_params = parse_qs(urlparse(str(data.link_profile)).query)
current_job_id = query_params.get("currentJobId", [None])[0]
jobs = api.get_job(current_job_id)
location = jobs['formattedLocation']
title = jobs['title']
listedAt = jobs['listedAt']
# Преобразование из миллисекунд в секунды и конвертация
future_date = datetime.fromtimestamp(listedAt / 1000)
# Текущая дата
current_date = datetime.now()
# Разница в днях
difference = abs((future_date - current_date).days)
# print(f"Разница в днях: {difference}")
jobPostingId = jobs['jobPostingId']
# workplaceTypesResolutionResults = jobs['workplaceTypesResolutionResults']
# Извлекаем все localizedName
# localized_names = [value['localizedName'] for value in workplaceTypesResolutionResults.values()]
# print(localized_names)
# localized_name = workplaceTypesResolutionResults['urn:li:fs_workplaceType:2']['localizedName']
link = f'https://www.linkedin.com/jobs/view/{current_job_id}/'
return {"job_id": current_job_id,
"job_title": title,
"minimum_annual_salary": f"minimum_annual_salary",
"salary_currency": 'salary_currency',
'location_type': "location_type",
'location': location,
"job_level": "job_level",
"job_type": 'job_type',
"days_posted": difference,
"hourly_rate": "hourly_rate",
"link": link}

192
routers/profile.py Normal file
View File

@ -0,0 +1,192 @@
from fastapi import FastAPI, APIRouter, Depends, Request, HTTPException, Form
from datetime import datetime
from dotenv import load_dotenv
import os
from fastapi.responses import HTMLResponse
from fastapi.responses import JSONResponse
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from pydantic import BaseModel, HttpUrl
from urllib.parse import urlparse, parse_qs
from linkedin_api import Linkedin
import logging
from utils.logging_setup import configure_global_logging
from models.models import async_session, Job
load_dotenv()
# # Ваши учетные данные LinkedIn
configure_global_logging()
router = APIRouter()
username = os.getenv('USERNAME')
password = os.getenv('PASSWD')
api = Linkedin(username, password)
DATABASE_URL = os.getenv('DATABASE_URL') # Ваши параметры подключения
engine = create_engine(DATABASE_URL)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
class UrlModel(BaseModel):
link_profile: HttpUrl # Используем HttpUrl для проверки валидности URL
async def get_session() -> AsyncSession:
async with async_session() as session:
yield session
async def get_or_create_profile(
db: AsyncSession, link: str, first_name: str, last_name: str, geoLocationName: str, all_skills: list, all_languages: list
):
# Проверяем, существует ли запись с таким first_name и last_name
query = select(Profile).filter(Profile.link == link)
result = await db.execute(query)
profile = result.scalars().first()
# Если профиля нет, создаём новый
if not profile:
profile = Profile(
link = link,
first_name=first_name,
last_name=last_name,
location=geoLocationName,
skills=all_skills,
languages=all_languages
)
db.add(profile)
await db.commit()
# await db.refresh(profile)
return profile
@router.post("/link_profile")
async def profile_url(data: UrlModel, db: AsyncSession = Depends(get_session)):
# Проверяем, что ссылка начинается с "https://xander"
if not str(data.link_profile).startswith("https://www.linkedin.com/"):
# Возвращаем исключение с кодом 400 и сообщением об ошибке
raise HTTPException(status_code=400, detail="Incorrect link")
url_profiles = str(data.link_profile)
path = urlparse(url_profiles).path
print(path)
# Пропускаем '/in/' и получаем последнюю часть
profiles = path.split("/")[2]
print(profiles)
profile = api.get_profile(profiles)
logging.info(f"{profile}")
firstName = profile['firstName']
lastName = profile['lastName']
locationName = profile['locationName']
geoLocationName = profile['geoLocationName']
headline = profile['headline']
educations = profile['education']
education = [
{
"schoolName": item.get("schoolName"),
"Field_of_Study": item.get("fieldOfStudy"),
"Degree": item.get('degreeName'),
"Grade": "??",
"Start Date": item.get("timePeriod", {}).get("startDate", {}).get("year"),
"End Date": item.get("timePeriod", {}).get("endDate", {}).get("year"),
"Currently study here": '??',
"Description": "??"
}
for item in educations
]
experiences = profile['experience']
experience = [
{
"companyName": item.get("companyName"),
"title": item.get("title"),
"Employment type": "??",
"Location": item.get('geoLocationName'),
"Location Type": "??",
"Start Date": {
"month": item.get("timePeriod", {}).get("startDate", {}).get("month"),
"year": item.get("timePeriod", {}).get("startDate", {}).get("year"),
},
"End Date":{
"month": item.get("timePeriod", {}).get("endDate", {}).get("month"),
"year": item.get("timePeriod", {}).get("endDate", {}).get("year"),
},
"Currently study here": '??',
"Description": item.get('description')
}
for item in experiences
]
volunteers = profile['volunteer']
volunteer = [
{
"Organization": item.get("companyName"),
"Role": item.get("role"),
"Cause": item.get("cause"),
"Location": item.get('geoLocationName'),
"Start Date": {
"month": item.get("timePeriod", {}).get("startDate", {}).get("month"),
"year": item.get("timePeriod", {}).get("startDate", {}).get("year"),
},
"End Date":{
"month": item.get("timePeriod", {}).get("endDate", {}).get("month"),
"year": item.get("timePeriod", {}).get("endDate", {}).get("year"),
},
"Currently study here": '??',
"Description": item.get('description')
}
for item in volunteers
]
all_skills = profile['skills']
skills = [item['name'] for item in all_skills]
all_languages = profile['languages']
languages = [item['name'] for item in all_languages]
profile = await get_or_create_profile(
db, data.link_profile, firstName, lastName, f"{geoLocationName} {locationName}", skills, languages
)
# Если профиль не в виде словаря, возвращаем его напрямую
return {"FirstName": firstName,
"LastName":lastName,
"Location": f"{geoLocationName} {locationName}",
"Statement":headline,
'Education - list': education,
'Experience - list': experience,
"Volunteering - list": volunteer,
"skills": skills,
"languages":languages}