diff --git a/.gitignore b/.gitignore index 4a2dcc6..1d918e0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,10 @@ node_modules/ # Игнорируем файлы виртуального окружения Python .venv/ .env/ +# Игнорируем папки для тестов и alembic test/ +alembic/ +alembic.ini # Игнорируем скомпилированные файлы *.out @@ -38,6 +41,7 @@ test/ /build/ /target/ + # Игнорируем файлы базы данных *.sqlite *.db diff --git a/README.md b/README.md index b37376b..3fb3801 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,13 @@ - [x] Профиль аккаунта по ссылке +- [x] Добавил модельку для работы - [ ] Вакансия по ссылке (частично) - [ ] Ежедневный обмен вакансий (Ожидаю доступы) > [!NOTE] -> Обмен по времени, один раз в сутки! +> Обмен по времени, один раз в сутки! Проверка объявлений до 10 дней в таблице Jobs столбик days_posted! > Пишет логи! diff --git a/envtemp b/envtemp index 0b81397..fb1d6e4 100644 --- a/envtemp +++ b/envtemp @@ -1,2 +1,3 @@ USERNAME= -PASSWD= \ No newline at end of file +PASSWD= +DATABASE_URL= \ No newline at end of file diff --git a/main.py b/main.py index b4fa0d7..f6d40d4 100644 --- a/main.py +++ b/main.py @@ -1,195 +1,33 @@ -from fastapi import FastAPI, HTTPException -from pydantic import BaseModel, HttpUrl -from urllib.parse import urlparse, parse_qs -from linkedin_api import Linkedin +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse import logging -from utils.logging_setup import configure_global_logging -from datetime import datetime -from dotenv import load_dotenv -import os - -load_dotenv() -# # Ваши учетные данные LinkedIn +from routers import profile, jobs -configure_global_logging() +# Инициализация приложения app = FastAPI( title="linkedin API", description="API с linkedin получаем данные по URL", - version="1.0.0" -) + version="1.0.0") -username = os.getenv('USERNAME') -password = os.getenv('PASSWD') -api = Linkedin(username, password) - -class Profile(BaseModel): - name: str - title: str - url: HttpUrl - -class UrlModel(BaseModel): - link_profile: HttpUrl # Используем HttpUrl для проверки валидности URL - -@app.post("/link_profile") -async def profile_url(data: UrlModel): - # Проверяем, что ссылка начинается с "https://xander" - if not str(data.link_profile).startswith("https://www.linkedin.com/"): - # Возвращаем исключение с кодом 400 и сообщением об ошибке - raise HTTPException(status_code=400, detail="Неправильная ссылка") - url_profiles = str(data.link_profile) - path = urlparse(url_profiles).path - print(path) - - # Пропускаем '/in/' и получаем последнюю часть - profiles = path.split("/")[2] - - print(profiles) - - profile = api.get_profile(profiles) - logging.info(f"{profile}") - firstName = profile['firstName'] - lastName = profile['lastName'] - locationName = profile['locationName'] - geoLocationName = profile['geoLocationName'] - headline = profile['headline'] - educations = profile['education'] - education = [ - { - "schoolName": item.get("schoolName"), - "Field_of_Study": item.get("fieldOfStudy"), - "Degree": item.get('degreeName'), - "Grade": "??", - "Start Date": item.get("timePeriod", {}).get("startDate", {}).get("year"), - "End Date": item.get("timePeriod", {}).get("endDate", {}).get("year"), - "Currently study here": '??', - "Description": "??" - - - } - for item in educations - ] - - experiences = profile['experience'] - experience = [ - { - "companyName": item.get("companyName"), - "title": item.get("title"), - "Employment type": "??", - "Location": item.get('geoLocationName'), - "Location Type": "??", - "Start Date": { - "month": item.get("timePeriod", {}).get("startDate", {}).get("month"), - "year": item.get("timePeriod", {}).get("startDate", {}).get("year"), - }, - "End Date":{ - "month": item.get("timePeriod", {}).get("endDate", {}).get("month"), - "year": item.get("timePeriod", {}).get("endDate", {}).get("year"), - }, - "Currently study here": '??', - "Description": item.get('description') - - - } - for item in experiences - ] - - volunteers = profile['volunteer'] - volunteer = [ - { - "Organization": item.get("companyName"), - "Role": item.get("role"), - "Cause": item.get("cause"), - "Location": item.get('geoLocationName'), - "Start Date": { - "month": item.get("timePeriod", {}).get("startDate", {}).get("month"), - "year": item.get("timePeriod", {}).get("startDate", {}).get("year"), - }, - "End Date":{ - "month": item.get("timePeriod", {}).get("endDate", {}).get("month"), - "year": item.get("timePeriod", {}).get("endDate", {}).get("year"), - }, - "Currently study here": '??', - "Description": item.get('description') - - - } - for item in volunteers - ] - all_skills = profile['skills'] - skills = [item['name'] for item in all_skills] - all_languages = profile['languages'] - languages = [item['name'] for item in all_languages] - - - - - # Если профиль не в виде словаря, возвращаем его напрямую - return {"FirstName": firstName, - "LastName":lastName, - "Location": f"{geoLocationName} {locationName}", - "Statement":headline, - 'Education - list': education, - 'Experience - list': experience, - "Volunteering - list": volunteer, - - - - "skills": skills, - "languages":languages} +# Подключение роутеров +app.include_router(profile.router, tags=["Profile"]) +app.include_router(jobs.router, tags=["Jobs"]) -@app.post("/link_vacancy") -async def vacancy_url(data: UrlModel): - if not str(data.link_profile).startswith("https://www.linkedin.com/"): - # Возвращаем исключение с кодом 400 и сообщением об ошибке - raise HTTPException(status_code=400, detail="Неправильная ссылка") - - query_params = parse_qs(urlparse(str(data.link_profile)).query) - current_job_id = query_params.get("currentJobId", [None])[0] +# Проверка состояния сервера +@app.get("/", tags=["Check"]) +async def check(): + return {"status": "ok"} - jobs = api.get_job(current_job_id) - location = jobs['formattedLocation'] - title = jobs['title'] - listedAt = jobs['listedAt'] - - # Преобразование из миллисекунд в секунды и конвертация - future_date = datetime.fromtimestamp(listedAt / 1000) - - # Текущая дата - current_date = datetime.now() - - # Разница в днях - difference = abs((future_date - current_date).days) - - # print(f"Разница в днях: {difference}") - - - jobPostingId = jobs['jobPostingId'] - # workplaceTypesResolutionResults = jobs['workplaceTypesResolutionResults'] - - - - # Извлекаем все localizedName - # localized_names = [value['localizedName'] for value in workplaceTypesResolutionResults.values()] - - # print(localized_names) - - # localized_name = workplaceTypesResolutionResults['urn:li:fs_workplaceType:2']['localizedName'] - link = f'https://www.linkedin.com/jobs/view/{current_job_id}/' - - - return {"job_id": current_job_id, - "job_title": title, - "minimum_annual_salary": f"minimum_annual_salary", - "salary_currency": 'salary_currency', - 'location_type': "location_type", - 'location': location, - "job_level": "job_level", - "job_type": 'job_type', - "days_posted": difference, - "hourly_rate": "hourly_rate", - "link": link} \ No newline at end of file +# Глобальный middleware для обработки необработанных исключений +@app.middleware("http") +async def catch_exceptions_middleware(request: Request, call_next): + try: + return await call_next(request) + except Exception as e: + logging.error(f"Unhandled exception: {str(e)}") + return JSONResponse(content={"error": "Internal Server Error"}, status_code=500) \ No newline at end of file diff --git a/models/models.py b/models/models.py new file mode 100644 index 0000000..2326958 --- /dev/null +++ b/models/models.py @@ -0,0 +1,68 @@ +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import sessionmaker, declarative_base +from sqlalchemy import Column, Integer, String, Text, DateTime, Float, Boolean +from sqlalchemy.dialects.mysql import JSON +from sqlalchemy.sql import func +from dotenv import load_dotenv +import os + +# Загрузка переменных окружения +load_dotenv() + +DATABASE_URL = os.getenv('DATABASE_URL') # Убедитесь, что URL настроен на асинхронный движок +# Например: "mysql+aiomysql://user:password@host:port/database" + +# Создание базы данных +Base = declarative_base() +engine = create_async_engine(DATABASE_URL, echo=True) +async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + +# Profile model +class Profile(Base): + __tablename__ = "profile" + + id = Column(Integer, primary_key=True, index=True) + link = Column(String(255), unique=True, nullable=True) + first_name = Column(String(255), nullable=False) + last_name = Column(String(255), nullable=False) + pronouns = Column(String(50), nullable=True) + email = Column(String(255), nullable=True) + + telephone = Column(String(20), nullable=True) + location = Column(String(255), nullable=True) + statement = Column(Text, nullable=True) + skills = Column(JSON, nullable=True) # List of skills + websites = Column(JSON, nullable=True) # List of website links + languages = Column(JSON, nullable=True) # List of languages with proficiency + date = Column(DateTime, server_default=func.now()) + +# Jobs model +class Job(Base): + __tablename__ = "jobs" + + job_id = Column(Integer, primary_key=True, index=True) + job_title = Column(String(255), nullable=False) + job_company = Column(String(255), nullable=False) + minimum_annual_salary = Column(Float, nullable=True) + salary_currency = Column(String(10), nullable=True) + location_type = Column(String(50), nullable=False) # Remote, On-site, Hybrid + location = Column(String(255), nullable=False) + job_level = Column(String(50), nullable=False) # Entry-level, Junior, Mid, etc. + job_type = Column(String(50), nullable=False) # Full-time, Part-time, etc. + days_posted = Column(Integer, nullable=False) + hourly_rate = Column(Float, nullable=True) + link = Column(String(2083), nullable=False) # URL for the job posting + link_company = Column(String(2083), nullable=False) # URL for the job posting + active = Column(Boolean, default=True) # Indicates if the job is active + +# Create database tables +async def init_db(): + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + +# Example for dependency injection in FastAPI +async def get_session(): + async with async_session() as session: + yield session + + diff --git a/models/start.py b/models/start.py new file mode 100644 index 0000000..db50027 --- /dev/null +++ b/models/start.py @@ -0,0 +1,10 @@ +import asyncio + +from models import init_db # Предполагается, что файл с моделями называется models.py + +async def main(): + await init_db() + print("Таблицы созданы!") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 81c931c..f252fb3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,29 @@ +aiomysql==0.2.0 +alembic==1.14.1 annotated-types==0.7.0 anyio==4.8.0 beautifulsoup4==4.12.3 certifi==2024.12.14 charset-normalizer==3.4.1 click==8.1.8 +databases==0.9.0 fastapi==0.115.6 +greenlet==3.1.1 h11==0.14.0 idna==3.10 -linkedin-api +linkedin-api @ git+https://github.com/tomquirk/linkedin-api.git@dacec3c9f03b4f1fbddb4f7dfdef57ea408e40aa lxml==5.3.0 +Mako==1.3.8 +MarkupSafe==3.0.2 pydantic==2.10.5 pydantic_core==2.27.2 +PyMySQL==1.1.1 python-dotenv==1.0.1 requests==2.32.3 +schedule==1.2.2 sniffio==1.3.1 soupsieve==2.6 +SQLAlchemy==2.0.37 starlette==0.41.3 typing_extensions==4.12.2 urllib3==2.3.0 diff --git a/routers/jobs.py b/routers/jobs.py new file mode 100644 index 0000000..81e8757 --- /dev/null +++ b/routers/jobs.py @@ -0,0 +1,107 @@ +from fastapi import FastAPI, APIRouter, Depends, Request, HTTPException, Form +from datetime import datetime +from dotenv import load_dotenv +import os + + +from fastapi.responses import HTMLResponse +from fastapi.responses import JSONResponse +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy.orm import sessionmaker +from sqlalchemy import create_engine +from sqlalchemy.orm import Session +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + + + + + +from pydantic import BaseModel, HttpUrl +from urllib.parse import urlparse, parse_qs +from linkedin_api import Linkedin +import logging +from utils.logging_setup import configure_global_logging + +load_dotenv() +# # Ваши учетные данные LinkedIn + + +configure_global_logging() + +router = APIRouter() + + +username = os.getenv('USERNAME') +password = os.getenv('PASSWD') +api = Linkedin(username, password) + + +DATABASE_URL = os.getenv('DATABASE_URL') # Ваши параметры подключения +engine = create_engine(DATABASE_URL) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + + + +class UrlModel(BaseModel): + link_profile: HttpUrl # Используем HttpUrl для проверки валидности URL + + +async def get_session() -> AsyncSession: + async with async_session() as session: + yield session + + + +@router.post("/link_vacancy") +async def vacancy_url(data: UrlModel): + if not str(data.link_profile).startswith("https://www.linkedin.com/"): + # Возвращаем исключение с кодом 400 и сообщением об ошибке + raise HTTPException(status_code=400, detail="Неправильная ссылка") + + query_params = parse_qs(urlparse(str(data.link_profile)).query) + current_job_id = query_params.get("currentJobId", [None])[0] + + jobs = api.get_job(current_job_id) + location = jobs['formattedLocation'] + title = jobs['title'] + listedAt = jobs['listedAt'] + + # Преобразование из миллисекунд в секунды и конвертация + future_date = datetime.fromtimestamp(listedAt / 1000) + + # Текущая дата + current_date = datetime.now() + + # Разница в днях + difference = abs((future_date - current_date).days) + + # print(f"Разница в днях: {difference}") + + + jobPostingId = jobs['jobPostingId'] + # workplaceTypesResolutionResults = jobs['workplaceTypesResolutionResults'] + + + + # Извлекаем все localizedName + # localized_names = [value['localizedName'] for value in workplaceTypesResolutionResults.values()] + + # print(localized_names) + + # localized_name = workplaceTypesResolutionResults['urn:li:fs_workplaceType:2']['localizedName'] + link = f'https://www.linkedin.com/jobs/view/{current_job_id}/' + + + return {"job_id": current_job_id, + "job_title": title, + "minimum_annual_salary": f"minimum_annual_salary", + "salary_currency": 'salary_currency', + 'location_type': "location_type", + 'location': location, + "job_level": "job_level", + "job_type": 'job_type', + "days_posted": difference, + "hourly_rate": "hourly_rate", + "link": link} \ No newline at end of file diff --git a/routers/profile.py b/routers/profile.py new file mode 100644 index 0000000..08c314e --- /dev/null +++ b/routers/profile.py @@ -0,0 +1,192 @@ +from fastapi import FastAPI, APIRouter, Depends, Request, HTTPException, Form +from datetime import datetime +from dotenv import load_dotenv +import os + + +from fastapi.responses import HTMLResponse +from fastapi.responses import JSONResponse +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy.orm import sessionmaker +from sqlalchemy import create_engine +from sqlalchemy.orm import Session +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + + + + + +from pydantic import BaseModel, HttpUrl +from urllib.parse import urlparse, parse_qs +from linkedin_api import Linkedin +import logging +from utils.logging_setup import configure_global_logging + +from models.models import async_session, Job + + +load_dotenv() +# # Ваши учетные данные LinkedIn + + +configure_global_logging() + +router = APIRouter() + + +username = os.getenv('USERNAME') +password = os.getenv('PASSWD') +api = Linkedin(username, password) + + +DATABASE_URL = os.getenv('DATABASE_URL') # Ваши параметры подключения +engine = create_engine(DATABASE_URL) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + + + +class UrlModel(BaseModel): + link_profile: HttpUrl # Используем HttpUrl для проверки валидности URL + + +async def get_session() -> AsyncSession: + async with async_session() as session: + yield session + + +async def get_or_create_profile( + db: AsyncSession, link: str, first_name: str, last_name: str, geoLocationName: str, all_skills: list, all_languages: list +): + # Проверяем, существует ли запись с таким first_name и last_name + query = select(Profile).filter(Profile.link == link) + result = await db.execute(query) + profile = result.scalars().first() + + # Если профиля нет, создаём новый + if not profile: + profile = Profile( + link = link, + first_name=first_name, + last_name=last_name, + location=geoLocationName, + skills=all_skills, + languages=all_languages + ) + db.add(profile) + await db.commit() + # await db.refresh(profile) + + return profile + + +@router.post("/link_profile") +async def profile_url(data: UrlModel, db: AsyncSession = Depends(get_session)): + # Проверяем, что ссылка начинается с "https://xander" + if not str(data.link_profile).startswith("https://www.linkedin.com/"): + # Возвращаем исключение с кодом 400 и сообщением об ошибке + raise HTTPException(status_code=400, detail="Incorrect link") + url_profiles = str(data.link_profile) + path = urlparse(url_profiles).path + print(path) + + # Пропускаем '/in/' и получаем последнюю часть + profiles = path.split("/")[2] + + print(profiles) + + profile = api.get_profile(profiles) + logging.info(f"{profile}") + firstName = profile['firstName'] + lastName = profile['lastName'] + locationName = profile['locationName'] + geoLocationName = profile['geoLocationName'] + headline = profile['headline'] + educations = profile['education'] + education = [ + { + "schoolName": item.get("schoolName"), + "Field_of_Study": item.get("fieldOfStudy"), + "Degree": item.get('degreeName'), + "Grade": "??", + "Start Date": item.get("timePeriod", {}).get("startDate", {}).get("year"), + "End Date": item.get("timePeriod", {}).get("endDate", {}).get("year"), + "Currently study here": '??', + "Description": "??" + + + } + for item in educations + ] + + experiences = profile['experience'] + experience = [ + { + "companyName": item.get("companyName"), + "title": item.get("title"), + "Employment type": "??", + "Location": item.get('geoLocationName'), + "Location Type": "??", + "Start Date": { + "month": item.get("timePeriod", {}).get("startDate", {}).get("month"), + "year": item.get("timePeriod", {}).get("startDate", {}).get("year"), + }, + "End Date":{ + "month": item.get("timePeriod", {}).get("endDate", {}).get("month"), + "year": item.get("timePeriod", {}).get("endDate", {}).get("year"), + }, + "Currently study here": '??', + "Description": item.get('description') + + + } + for item in experiences + ] + + volunteers = profile['volunteer'] + volunteer = [ + { + "Organization": item.get("companyName"), + "Role": item.get("role"), + "Cause": item.get("cause"), + "Location": item.get('geoLocationName'), + "Start Date": { + "month": item.get("timePeriod", {}).get("startDate", {}).get("month"), + "year": item.get("timePeriod", {}).get("startDate", {}).get("year"), + }, + "End Date":{ + "month": item.get("timePeriod", {}).get("endDate", {}).get("month"), + "year": item.get("timePeriod", {}).get("endDate", {}).get("year"), + }, + "Currently study here": '??', + "Description": item.get('description') + + + } + for item in volunteers + ] + all_skills = profile['skills'] + skills = [item['name'] for item in all_skills] + all_languages = profile['languages'] + languages = [item['name'] for item in all_languages] + profile = await get_or_create_profile( + db, data.link_profile, firstName, lastName, f"{geoLocationName} {locationName}", skills, languages + ) + + + + + # Если профиль не в виде словаря, возвращаем его напрямую + return {"FirstName": firstName, + "LastName":lastName, + "Location": f"{geoLocationName} {locationName}", + "Statement":headline, + 'Education - list': education, + 'Experience - list': experience, + "Volunteering - list": volunteer, + + + + "skills": skills, + "languages":languages} \ No newline at end of file