Merge branch 'develope'

* develope: (172 commits)
  update k8s_namespace_waiter add timeout logger
  Add [PUT] /maintenance/update_pj_http_url API for update project table http_url and ssh_url.
  *Update k8sClient.py, sync_project.py 1. add get_namespace 2. add k8s_namespace_waiter 3. add rc_add_namespace_into_rc_project 4. fixed bot_process bot name
  fixed update_issue check fixed_version exist first
  fixed update_issue args fixed_version_id key error
  update_issue fixed_versions_id support null
  *Update apiError.py, issue.py 1. add invalid_fixed_version_id 2. update and create issue need to check fixed_version status
  Ignore ingress subdomain when service was 'db-gui'
  *Update sync_project.py, sonarqube.py 1. update sq_list_project params 2. fixed check_hb_pj get X-Total-Count 3. refactor check_sq_pj
  *Update sync_project.py 1. refactor most functions 2. auto add members by project_user_role
  sonarqube add sq_list_user, sq_list_member
  k8sClient add list_role_binding_in_namespace
  harbor add hb_list_user, hb_list_member
  gitlab add gl_project_list_member
  Use http when used nodePort
  API add response git tag and git commit time
  Generate git tag, git commit time.
  Remove redundant code
  Check ingreess exist when api generate service URL
  Add force_id to version list
  ...

.gitignore

@@ -9,4 +9,5 @@ __pycache__
 alembic.ini
 .api_version
 pj_push_template
-pj_edit_pipe_yaml
+pj_edit_pipe_yaml
+.vscode

Dockerfile

@@ -4,6 +4,9 @@ COPY . .
 RUN mkdir /root/.kube
 COPY k8s_config /root/.kube/config
 RUN git rev-parse HEAD > git_commit
+RUN git fetch origin master:master
+RUN git describe --tags `git rev-list --tags --max-count=1` > git_tag
+RUN git log -1 --date=iso8601 --format="%ad" > git_date
 RUN pip install --no-cache-dir -r requirements.txt 
 RUN python -c 'import sys, yaml, json; yaml.safe_dump(json.load(sys.stdin), sys.stdout, default_flow_style=False)' < k8s_config > ~/.kube/config
 #CMD [ "python", "apis/api.py"]

_environments.json

@@ -39,6 +39,9 @@
     "HARBOR_ACCOUNT": "admin",
     "HARBOR_PASSWORD": "***********",
 
+    "INGRESS_EXTERNAL_BASE": "ingress-dev.iiidevops.org",
+	"INGRESS_EXTERNAL_TLS": "ingress-tls",
+
     "ADMIN_INIT_LOGIN": "devops_admin",
     "ADMIN_INIT_EMAIL": "[email protected]",
     "ADMIN_INIT_PASSWORD": "********",

apis/alembic/versions/1ebcdf2b64ba_add_issuecollectionrelation_table.py

@@ -0,0 +1,38 @@
+"""add IssueCollectionRelation table
+
+Revision ID: 1ebcdf2b64ba
+Revises: b2195255b274
+Create Date: 2021-05-27 09:02:48.983611
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1ebcdf2b64ba'
+down_revision = 'b2195255b274'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('issue_collection_relation',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('project_id', sa.Integer(), nullable=True),
+    sa.Column('issue_id', sa.Integer(), nullable=True),
+    sa.Column('software_name', sa.String(), nullable=True),
+    sa.Column('file_name', sa.String(), nullable=True),
+    sa.Column('plan_name', sa.String(), nullable=True),
+    sa.Column('items', sa.JSON(), nullable=True),
+    sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('issue_collection_relation')
+    # ### end Alembic commands ###

apis/alembic/versions/7707fc9fbdde_update_registries_remove_cloudprovider.py

@@ -0,0 +1,50 @@
+"""update registries remove cloudprovider
+
+Revision ID: 7707fc9fbdde
+Revises: 1ebcdf2b64ba
+Create Date: 2021-05-28 09:41:33.958676
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '7707fc9fbdde'
+down_revision = '1ebcdf2b64ba'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('registries', sa.Column('access_key', sa.String(), nullable=True))
+    op.add_column('registries', sa.Column('access_secret', sa.String(), nullable=True))
+    op.add_column('registries', sa.Column('description', sa.String(), nullable=True))
+    op.add_column('registries', sa.Column('type', sa.String(), nullable=True))
+    op.add_column('registries', sa.Column('url', sa.String(), nullable=True))
+    op.drop_constraint('registries_provider_id_fkey', 'registries', type_='foreignkey')
+    op.drop_column('registries', 'provider_id')
+    op.drop_table('cloud_provider')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('registries', sa.Column('provider_id', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.drop_column('registries', 'url')
+    op.drop_column('registries', 'type')
+    op.drop_column('registries', 'description')
+    op.drop_column('registries', 'access_secret')
+    op.drop_column('registries', 'access_key')
+    op.create_table('cloud_provider',
+    sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
+    sa.Column('name', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=True),
+    sa.Column('type', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('provider_info', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], name='cloud_provider_user_id_fkey', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id', name='cloud_provider_pkey')
+    )
+    op.create_foreign_key('registries_provider_id_fkey', 'registries', 'cloud_provider', ['provider_id'], ['id'], ondelete='CASCADE')
+    # ### end Alembic commands ###

apis/alembic/versions/987ed3077e84_add_plugin_software_type_id_column.py

@@ -0,0 +1,28 @@
+"""add plugin_software_type_id column
+
+Revision ID: 987ed3077e84
+Revises: 7707fc9fbdde
+Create Date: 2021-06-07 10:58:09.196144
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '987ed3077e84'
+down_revision = '7707fc9fbdde'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('plugin_software', sa.Column('type_id', sa.Integer(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('plugin_software', 'type_id')
+    # ### end Alembic commands ###

apis/alembic/versions/c296a8ba63ac_add_test_generated_issue_table.py

@@ -0,0 +1,40 @@
+"""Add test generated issue table
+
+Revision ID: c296a8ba63ac
+Revises: 987ed3077e84
+Create Date: 2021-06-21 16:19:59.125357
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c296a8ba63ac'
+down_revision = '987ed3077e84'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('test_generated_issue',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('project_id', sa.Integer(), nullable=True),
+    sa.Column('issue_id', sa.Integer(), nullable=True),
+    sa.Column('software_name', sa.String(), nullable=True),
+    sa.Column('file_name', sa.String(), nullable=True),
+    sa.Column('branch', sa.String(), nullable=True),
+    sa.Column('commit_id', sa.String(), nullable=True),
+    sa.Column('result_table', sa.String(), nullable=False),
+    sa.Column('result_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('test_generated_issue')
+    # ### end Alembic commands ###

apis/alembic/versions/e55508cb2d34_add_release_table_for_release_record.py

@@ -0,0 +1,42 @@
+"""Add Release Table for release record
+
+Revision ID: e55508cb2d34
+Revises: c296a8ba63ac
+Create Date: 2021-06-24 16:54:48.451314
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'e55508cb2d34'
+down_revision = 'c296a8ba63ac'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('release',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('version_id', sa.Integer(), nullable=True),
+    sa.Column('versions', sa.String(), nullable=True),
+    sa.Column('issues', sa.String(), nullable=True),
+    sa.Column('branch', sa.String(), nullable=True),
+    sa.Column('commit', sa.String(), nullable=True),
+    sa.Column('tag_name', sa.String(), nullable=True),
+    sa.Column('note', sa.String(), nullable=True),
+    sa.Column('creator_id', sa.Integer(), nullable=True),
+    sa.Column('create_at', sa.DateTime(), nullable=True),
+    sa.Column('update_at', sa.DateTime(), nullable=True),
+    sa.ForeignKeyConstraint(['creator_id'], ['user.id'], ondelete='SET NULL'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('release')
+    # ### end Alembic commands ###

apis/api.py

@@ -30,7 +30,7 @@
 from model import db
 from resources import logger, role as role, activity, zap, sideex
 from resources import project, gitlab, issue, user, redmine, wiki, version, sonarqube, apiTest, postman, mock, harbor, \
-    webInspect, template, release, sync_redmine, plugin, kubernetesClient, ad, project_permission, quality
+    webInspect, template, release, sync_redmine, plugin, kubernetesClient, ad, project_permission, quality, sync_project
 
 app = Flask(__name__)
 for key in ['JWT_SECRET_KEY',
@@ -84,12 +84,20 @@ def internal_error(exception):
 # noinspection PyMethodMayBeStatic
 class SystemGitCommitID(Resource):
     def get(self):
+        git_commit_id = ""
+        git_tag = ""
+        git_date = ""
         if os.path.exists("git_commit"):
             with open("git_commit") as f:
                 git_commit_id = f.read().splitlines()[0]
-                return util.success({"git_commit_id": "{0}".format(git_commit_id)})
-        else:
-            raise apiError.DevOpsError(400, "git_commit file is not exist.")
+        if os.path.exists("git_tag"):
+            with open("git_tag") as f:
+                git_tag = f.read().splitlines()[0]
+        if os.path.exists("git_date"):
+            with open("git_date") as f:
+                git_date = f.read().splitlines()[0]
+        return util.success({"git_commit_id": git_commit_id, "git_tag": git_tag, "git_date": git_date})
+
 
 
 class NexusVersion(Resource):
@@ -182,6 +190,7 @@ def initialize(db_uri):
 
 # Projects
 api.add_resource(project.ListMyProjects, '/project/list')
+api.add_resource(project.ListProjectsByUser, '/projects_by_user/<int:user_id>')
 api.add_resource(project.SingleProject, '/project', '/project/<sint:project_id>')
 api.add_resource(project.SingleProjectByName, '/project_by_name/<project_name>')
 api.add_resource(project.ProjectUserList, '/project/<sint:project_id>/user/list')
@@ -274,9 +283,10 @@ def initialize(db_uri):
 socketio.on_namespace(rancher.RancherWebsocketLog('/rancher/websocket/logs'))
 
 
-
 # issue
 api.add_resource(issue.IssueByProject, '/project/<sint:project_id>/issues')
+api.add_resource(issue.IssueFamily, '/issue/<issue_id>/family')
+api.add_resource(issue.IssueListByProject, '/project/<sint:project_id>/issues_list')
 api.add_resource(issue.IssueByTreeByProject, '/project/<sint:project_id>/issues_by_tree')
 api.add_resource(issue.IssueByStatusByProject,
                  '/project/<sint:project_id>/issues_by_status')
@@ -295,6 +305,8 @@ def initialize(db_uri):
 api.add_resource(issue.MyOpenIssueStatistics, '/issues/open_statistics')
 api.add_resource(issue.MyIssueWeekStatistics, '/issues/week_statistics')
 api.add_resource(issue.MyIssueMonthStatistics, '/issues/month_statistics')
+api.add_resource(issue.Relation, '/issues/relation', '/issues/relation/<int:relation_id>')
+api.add_resource(issue.CheckIssueClosable, '/issues/<issue_id>/check_closable')
 
 # Release
 api.add_resource(release.Releases, '/project/<project_id>/releases')
@@ -428,6 +440,7 @@ def initialize(db_uri):
                  '/maintenance/secretes_into_rc_all/<secret_name>')
 api.add_resource(maintenance.RegistryIntoRcAll, '/maintenance/registry_into_rc_all',
                  '/maintenance/registry_into_rc_all/<registry_name>')
+api.add_resource(maintenance.UpdatePjHttpUrl, '/maintenance/update_pj_http_url')
 
 # Rancher
 api.add_resource(rancher.Catalogs, '/rancher/catalogs')
@@ -455,11 +468,21 @@ def initialize(db_uri):
 api.add_resource(project_permission.SetPermission, '/project_permission/set_permission')
 
 # Quality
-api.add_resource(quality.Collection, '/quality/<int:repository_id>/collection')
+api.add_resource(quality.TestPlanList, '/quality/<int:project_id>/testplan_list')
+api.add_resource(quality.TestPlan, '/quality/<int:project_id>/testplan/<int:testplan_id>')
+api.add_resource(quality.TestFileList, '/quality/<int:project_id>/testfile_list')
+api.add_resource(quality.TestFile, '/quality/<int:project_id>/testfile', 
+                 '/quality/<int:project_id>/testfile/<test_file_name>')
+api.add_resource(quality.TestPlanWithTestFile, '/quality/<int:project_id>/testplan_with_testfile',
+                 '/quality/<int:project_id>/testplan_with_testfile/<int:item_id>')
+
 
 # System versions
 api.add_resource(NexusVersion, '/system_versions')
 
+# Sync Projects
+api.add_resource(sync_project.SyncProject, '/sync_projects')
+
 
 def start_prod():
     try:
@@ -481,4 +504,4 @@ def start_prod():
 if __name__ == "__main__":
     start_prod()
     socketio.run(app, host='0.0.0.0', port=10009, debug=(config.get('DEBUG')),
-                 use_reloader=True)
+                 use_reloader=False)