Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

added flag -c contributors logging #58

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,10 @@ python3 main.py --invites [-t, --token] token (github токен вместо to
```commandline
python3 main.py [-w, --wikis] [-t, --token] token (github токен вместо token) [-l, --list] list (list - строка пути к txt файлу со списком репозиториев) --dowland_repos path_drepo (path_drepo - строка пути к директории, где сохраняются вики-репозитории) [-o, --out] out (out - название csv файла, в который будут помещены все логи)
```

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

может тут вернуть?

6. Логирование contributors
```commandline
python3 main.py [--contributors] [-t, --token] token (github токен вместо token) [-l, --list] list (list - строка пути к txt файлу со списком репозиториев) [-o, --out] out (out - название csv файла, в который будут помещены все логи)
```

## Получение токена для работы с Google таблицей:
Сначала нужно создать проект на сайте [Google Cloud](https://console.cloud.google.com/). Выбираем название проекта, жмем на кнопку "Create".
Expand Down
81 changes: 81 additions & 0 deletions contributors_parser.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
import csv
from time import sleep
from github import Github, Repository
import requests

EMPTY_FIELD = 'Empty field'
TIMEDELTA = 0.05
TIMEZONE = 'Europe/Moscow'
FIELDNAMES = ('repository name', 'login', 'name', 'email', 'url', 'permissions', 'total_commits', 'id', 'node_id', 'type', 'bio', 'site_admin')
Comment on lines +6 to +9
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

в другом ПРе выносится это в отдельный файлик
главное потом не забыть вынести)



def log_contributors_to_csv(info:dict, csv_name:str):
with open(csv_name, 'a', newline='') as file:
writer = csv.DictWriter(file, fieldnames=FIELDNAMES)
writer.writerow(info)


def log_repository_contributors(repository: Repository, csv_name:str, token:str):

contributors = repository.get_contributors()
total_commits_dict = get_contributor_commits(repository.owner, repository.name, token)
nvl = lambda val: val or EMPTY_FIELD
for contributor in contributors:
contributor_permissons = repository.get_collaborator_permission(contributor)
contributor_total_commits = total_commits_dict.get(contributor.login, EMPTY_FIELD)

info_tmp = {
'repository name': repository.full_name,
'login': contributor.login,
'name': nvl(contributor.name),
'email': nvl(contributor.email),
'url': contributor.html_url,
'permissions': nvl(contributor_permissons),
'total_commits': contributor_total_commits,
'id': contributor.id,
'node_id': contributor.node_id,
'type': contributor.type,
'bio': nvl(contributor.bio),
'site_admin': contributor.site_admin,
}
log_contributors_to_csv(info_tmp, csv_name)
Comment on lines +27 to +41
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

и такой подход планируется заменить на датаклассы
#54

print(info_tmp)
sleep(TIMEDELTA)


def log_contributors(client: Github, token:str, working_repos:list, csv_name:str, fork_flag:str):
with open(csv_name, 'w', newline='') as file:
writer = csv.writer(file)
writer.writerow(FIELDNAMES)

for repo in working_repos:
try:
print('=' * 20, repo.full_name, '=' * 20)
log_repository_contributors(repo, csv_name,token)
if fork_flag:
for forked_repo in repo.get_forks():
print('=' * 20, "FORKED:", forked_repo.full_name, '=' * 20)
log_repository_contributors(forked_repo, csv_name)
except Exception as e:
print(e)

def get_contributor_commits(repo_owner, repo_name, token):
headers = {"Authorization": f"Bearer {token}"}
request_name = f"https://api.github.com/repos/{repo_owner.login}/{repo_name}/stats/contributors"
request = requests.get(request_name, headers=headers)

while request.status_code == 202:
print("Waiting for response...")
sleep(10)
request = requests.get(request_name, headers=headers)
Comment on lines +67 to +70
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

А что за код 202? почему мы его ретраим?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Кажется лучше сделать какой-то порог по количеству ретраев и если не получилось за условные 5 попыток, то падать
Вечно ждать никто не будет :(


if request.status_code != 200:
return {}

response_data = request.json()
total_commits_dict = {}
for contributor in response_data:
contributor_name = contributor["author"]["login"]
total_commits = contributor["total"]
total_commits_dict[contributor_name] = total_commits
return total_commits_dict
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

добавить пустую строку

5 changes: 5 additions & 0 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import issues_parser
import invites_parser
import wikipars
import contributors_parser

def parse_args():
parser = argparse.ArgumentParser()
Expand All @@ -17,6 +18,7 @@ def parse_args():
parser.add_argument("-p", "--pull_requests", help="log pull requests", action="store_true")
parser.add_argument("-i", "--issues", help="log issues", action="store_true")
parser.add_argument("-w", "--wikis", help="log wikis", action="store_true")
parser.add_argument("--contributors", help="log contributors", action="store_true")
parser.add_argument("--forks_include", help="logging data from forks", action="store_true")
parser.add_argument("-e", "--export_google_sheets", help="export table to google sheets", action="store_true")
parser.add_argument('-t', '--token', type=str, required=True, help='token github account')
Expand Down Expand Up @@ -81,9 +83,12 @@ def main():
invites_parser.log_invitations(client, working_repos, csv_name)
if args.wikis:
wikipars.wikiparser(client, repositories, path_drepo, csv_name)
if args.contributors:
contributors_parser.log_contributors(client, token, working_repos, csv_name, fork_flag)
if args.export_google_sheets:
export_sheets.write_data_to_table(csv_name, args.google_token, args.table_id, args.sheet_id)


if __name__ == '__main__':
main()

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

а тут убрать