diff --git a/backend/kernelCI_app/helpers/issueExtras.py b/backend/kernelCI_app/helpers/issueExtras.py index 82647bc61..ee23d6b55 100644 --- a/backend/kernelCI_app/helpers/issueExtras.py +++ b/backend/kernelCI_app/helpers/issueExtras.py @@ -4,6 +4,7 @@ from kernelCI_app.constants.general import UNCATEGORIZED_STRING from kernelCI_app.helpers.logger import log_message from kernelCI_app.queries.issues import ( + get_issue_first_good_checkouts, get_issue_first_seen_data, get_issue_last_seen_data, get_issue_trees_data, @@ -11,8 +12,10 @@ from kernelCI_app.typeModels.issues import ( ExtraIssuesData, Incident, + IssueCheckout, IssueWithExtraInfo, ProcessedExtraDetailedIssues, + TreeSeenData, TreeSetItem, ) @@ -46,6 +49,26 @@ def _incident_from_record(record: dict) -> Incident: ) +def _checkout_from_record(record: dict) -> IssueCheckout: + return IssueCheckout( + start_time=record["start_time"], + git_commit_hash=record["git_commit_hash"], + git_repository_url=record["git_repository_url"], + git_repository_branch=record["git_repository_branch"], + git_commit_name=record["git_commit_name"], + tree_name=record["tree_name"], + checkout_id=record["checkout_id"], + ) + + +def _tree_key(record: dict) -> tuple: + return ( + record["tree_name"], + record["git_repository_url"], + record["git_repository_branch"], + ) + + def process_issues_extra_details( *, issue_key_list: List[Tuple[str, int]], @@ -105,6 +128,67 @@ def assign_issue_incidents( processed_issue_from_id.versions.setdefault(version, None) +def assign_issue_per_tree( + *, + issue_key_list: List[Tuple[str, int]], + processed_issues_table: ProcessedExtraDetailedIssues, +) -> None: + """Assigns per-tree first/last seen and first good checkout to each issue.""" + issue_id_set = {issue_id for issue_id, _ in issue_key_list} + + first_records = get_issue_first_seen_data( + issue_id_list=list(issue_id_set), group_by="tree" + ) + last_records = get_issue_last_seen_data( + issue_id_list=list(issue_id_set), group_by="tree" + ) + + first_by_tree: dict[str, dict[tuple, dict]] = defaultdict(dict) + for record in first_records: + first_by_tree[record["issue_id"]][_tree_key(record)] = record + + last_by_tree: dict[str, dict[tuple, dict]] = defaultdict(dict) + for record in last_records: + last_by_tree[record["issue_id"]][_tree_key(record)] = record + + tree_seen_by_key: dict[tuple[str, tuple], TreeSeenData] = {} + last_seen_trees: list[dict] = [] + + for issue_id, last_map in last_by_tree.items(): + if issue_id not in processed_issues_table: + continue + + trees: list[TreeSeenData] = [] + for tree_key, last_record in last_map.items(): + first_record = first_by_tree[issue_id].get(tree_key, last_record) + tree_seen = TreeSeenData( + first_incident=_incident_from_record(first_record), + last_incident=_incident_from_record(last_record), + ) + trees.append(tree_seen) + tree_seen_by_key[(issue_id, tree_key)] = tree_seen + + last_seen_trees.append( + { + "issue_id": issue_id, + "tree_name": last_record["tree_name"], + "git_repository_url": last_record["git_repository_url"], + "git_repository_branch": last_record["git_repository_branch"], + "last_seen": last_record.get("checkout_start_time") + or last_record["first_seen"], + } + ) + + processed_issues_table[issue_id].per_tree = trees + + for good_record in get_issue_first_good_checkouts(last_seen_trees=last_seen_trees): + tree_seen = tree_seen_by_key.get( + (good_record["issue_id"], _tree_key(good_record)) + ) + if tree_seen is not None: + tree_seen.first_good_checkout = _checkout_from_record(good_record) + + def assign_issue_trees( *, issue_key_list: list[tuple[str, int]], diff --git a/backend/kernelCI_app/queries/issues.py b/backend/kernelCI_app/queries/issues.py index 359d5ca6b..2fd8484ca 100644 --- a/backend/kernelCI_app/queries/issues.py +++ b/backend/kernelCI_app/queries/issues.py @@ -240,57 +240,104 @@ def get_test_issues(*, test_id: str) -> list[dict]: def get_issue_seen_data( - *, issue_id_list: list[str], mode: Literal["first", "last"] = "first" + *, + issue_id_list: list[str], + mode: Literal["first", "last"] = "first", + group_by: Literal["issue", "tree"] = "issue", ) -> list[dict]: """ Retrieves the incident and checkout data of either the first or last incident of a list of issues through a list of `issue_id`s. :param mode: Either 'first' to get oldest incidents or 'last' to get the newest ones. + :param group_by: 'issue' returns one row per issue; 'tree' returns one row per + (issue, tree_name, git_repository_url, git_repository_branch). """ if not issue_id_list: return [] order_direction = "ASC" if mode == "first" else "DESC" cache_key = f"issue_{mode}_seen" + if group_by == "tree": + cache_key = f"{cache_key}_per_tree" params = {"issue_id_list": issue_id_list} records = get_query_cache(key=cache_key, params=params) if records is None: - query = f""" - WITH target_incident AS ( - SELECT DISTINCT ON (IC.issue_id) - IC.id + if group_by == "tree": + query = f""" + SELECT DISTINCT ON ( + IC.issue_id, + C.tree_name, + C.git_repository_url, + C.git_repository_branch + ) + IC.id, + IC.issue_id, + IC._timestamp AS first_seen, + IC.issue_version, + C.git_commit_hash, + C.git_repository_url, + C.git_repository_branch, + C.git_commit_name, + C.tree_name, + C.id AS checkout_id, + C.start_time AS checkout_start_time FROM incidents IC + LEFT JOIN tests T ON IC.test_id = T.id + LEFT JOIN builds B ON ( + IC.build_id = B.id + OR T.build_id = B.id + ) + LEFT JOIN checkouts C ON B.checkout_id = C.id WHERE IC.issue_id = ANY(%(issue_id_list)s) + AND C.tree_name IS NOT NULL + AND C.git_repository_branch IS NOT NULL ORDER BY IC.issue_id, + C.tree_name, + C.git_repository_url, + C.git_repository_branch, IC.issue_version {order_direction}, IC._timestamp {order_direction} - ) - SELECT - IC.id, - IC.issue_id, - IC._timestamp AS first_seen, - IC.issue_version, - C.git_commit_hash, - C.git_repository_url, - C.git_repository_branch, - C.git_commit_name, - C.tree_name, - C.id as checkout_id - FROM - incidents IC - LEFT JOIN tests T ON IC.test_id = T.id - LEFT JOIN builds B ON ( - IC.build_id = B.id - OR T.build_id = B.id - ) - LEFT JOIN checkouts C ON B.checkout_id = C.id - JOIN target_incident TI ON IC.id = TI.id - """ + """ + else: + query = f""" + WITH target_incident AS ( + SELECT DISTINCT ON (IC.issue_id) + IC.id + FROM + incidents IC + WHERE + IC.issue_id = ANY(%(issue_id_list)s) + ORDER BY + IC.issue_id, + IC.issue_version {order_direction}, + IC._timestamp {order_direction} + ) + SELECT + IC.id, + IC.issue_id, + IC._timestamp AS first_seen, + IC.issue_version, + C.git_commit_hash, + C.git_repository_url, + C.git_repository_branch, + C.git_commit_name, + C.tree_name, + C.id as checkout_id + FROM + incidents IC + LEFT JOIN tests T ON IC.test_id = T.id + LEFT JOIN builds B ON ( + IC.build_id = B.id + OR T.build_id = B.id + ) + LEFT JOIN checkouts C ON B.checkout_id = C.id + JOIN target_incident TI ON IC.id = TI.id + """ with connection.cursor() as cursor: cursor.execute(query, params) @@ -301,22 +348,128 @@ def get_issue_seen_data( return records -def get_issue_first_seen_data(*, issue_id_list: list[str]) -> list[dict]: +def get_issue_first_seen_data( + *, + issue_id_list: list[str], + group_by: Literal["issue", "tree"] = "issue", +) -> list[dict]: """ Retrieves the incident and checkout data of the first incident of a list of issues through a list of `issue_id`s. """ - return get_issue_seen_data(issue_id_list=issue_id_list, mode="first") + return get_issue_seen_data( + issue_id_list=issue_id_list, mode="first", group_by=group_by + ) -def get_issue_last_seen_data(*, issue_id_list: list[str]) -> list[dict]: +def get_issue_last_seen_data( + *, + issue_id_list: list[str], + group_by: Literal["issue", "tree"] = "issue", +) -> list[dict]: """ Retrieves the incident and checkout data of the last incident of a list of issues through a list of `issue_id`s. """ - return get_issue_seen_data(issue_id_list=issue_id_list, mode="last") + return get_issue_seen_data( + issue_id_list=issue_id_list, mode="last", group_by=group_by + ) + + +def get_issue_first_good_checkouts( + *, + last_seen_trees: list[dict], +) -> list[dict]: + """ + For each (issue_id, tree, url, branch, last_seen) entry, returns the earliest + checkout on that tree with start_time > last_seen and no incident for the issue. + + Uses a single query with a LATERAL join so PostgreSQL does one indexed lookup + per tree instead of scanning every later checkout for every tree. + + :param last_seen_trees: dicts with keys issue_id, tree_name, git_repository_url, + git_repository_branch, last_seen (datetime). + """ + if not last_seen_trees: + return [] + + params = { + "issue_ids": [entry["issue_id"] for entry in last_seen_trees], + "tree_names": [entry["tree_name"] for entry in last_seen_trees], + "git_repository_urls": [ + entry["git_repository_url"] for entry in last_seen_trees + ], + "git_repository_branches": [ + entry["git_repository_branch"] for entry in last_seen_trees + ], + "last_seens": [entry["last_seen"] for entry in last_seen_trees], + } + + query = """ + WITH last_seen_trees AS ( + SELECT * + FROM unnest( + %(issue_ids)s::text[], + %(tree_names)s::text[], + %(git_repository_urls)s::text[], + %(git_repository_branches)s::text[], + %(last_seens)s::timestamptz[] + ) AS t( + issue_id, + tree_name, + git_repository_url, + git_repository_branch, + last_seen + ) + ) + SELECT + ls.issue_id, + good.checkout_id, + good.start_time, + good.git_commit_hash, + good.git_repository_url, + good.git_repository_branch, + good.git_commit_name, + good.tree_name + FROM + last_seen_trees ls + CROSS JOIN LATERAL ( + SELECT + C.id AS checkout_id, + C.start_time, + C.git_commit_hash, + C.git_repository_url, + C.git_repository_branch, + C.git_commit_name, + C.tree_name + FROM checkouts C + WHERE + C.tree_name = ls.tree_name + AND C.git_repository_url IS NOT DISTINCT FROM ls.git_repository_url + AND C.git_repository_branch = ls.git_repository_branch + AND C.start_time > ls.last_seen + AND NOT EXISTS ( + SELECT 1 + FROM incidents IC + LEFT JOIN tests T ON IC.test_id = T.id + LEFT JOIN builds B ON ( + IC.build_id = B.id + OR T.build_id = B.id + ) + WHERE + IC.issue_id = ls.issue_id + AND B.checkout_id = C.id + ) + ORDER BY C.start_time ASC + LIMIT 1 + ) good + """ + + with connection.cursor() as cursor: + cursor.execute(query, params) + return dict_fetchall(cursor) def get_issue_trees_data( diff --git a/backend/kernelCI_app/typeModels/issues.py b/backend/kernelCI_app/typeModels/issues.py index 878150cf9..806d61784 100644 --- a/backend/kernelCI_app/typeModels/issues.py +++ b/backend/kernelCI_app/typeModels/issues.py @@ -78,10 +78,27 @@ class Incident(BaseModel): checkout_id: Optional[str] +class IssueCheckout(BaseModel): + start_time: Timestamp + git_commit_hash: Optional[Checkout__GitCommitHash] = None + git_repository_url: Optional[Checkout__GitRepositoryUrl] = None + git_repository_branch: Optional[Checkout__GitRepositoryBranch] = None + git_commit_name: Optional[Checkout__GitCommitName] = None + tree_name: Optional[Checkout__TreeName] = None + checkout_id: Optional[str] = None + + +class TreeSeenData(BaseModel): + first_incident: Incident + last_incident: Incident + first_good_checkout: Optional[IssueCheckout] = None + + class ExtraIssuesData(BaseModel): first_incident: Incident last_incident: Incident versions: dict[int, Optional[IssueWithExtraInfo]] + per_tree: Optional[list[TreeSeenData]] = None type ProcessedExtraDetailedIssues = Annotated[ diff --git a/backend/kernelCI_app/views/issueDetailsView.py b/backend/kernelCI_app/views/issueDetailsView.py index 3c7a08c04..2306c1b12 100644 --- a/backend/kernelCI_app/views/issueDetailsView.py +++ b/backend/kernelCI_app/views/issueDetailsView.py @@ -8,7 +8,10 @@ from kernelCI_app.constants.localization import ClientStrings from kernelCI_app.helpers.errorHandling import create_api_error_response -from kernelCI_app.helpers.issueExtras import process_issues_extra_details +from kernelCI_app.helpers.issueExtras import ( + assign_issue_per_tree, + process_issues_extra_details, +) from kernelCI_app.queries.issues import get_issue_details, get_latest_issue_version from kernelCI_app.typeModels.commonOpenApiParameters import ISSUE_ID_PATH_PARAM from kernelCI_app.typeModels.issueDetails import ( @@ -61,6 +64,10 @@ def get(self, _request, issue_id: Optional[str]) -> Response: issue_key_list=[(issue_id, query_params.version)], processed_issues_table=self.processed_issue_extras, ) + assign_issue_per_tree( + issue_key_list=[(issue_id, query_params.version)], + processed_issues_table=self.processed_issue_extras, + ) try: valid_response = IssueDetailsResponse( diff --git a/backend/schema.yml b/backend/schema.yml index fe12505b9..a11f41322 100644 --- a/backend/schema.yml +++ b/backend/schema.yml @@ -2513,6 +2513,14 @@ components: - type: 'null' title: Versions type: object + per_tree: + anyOf: + - items: + $ref: '#/components/schemas/TreeSeenData' + type: array + - type: 'null' + default: null + title: Per Tree required: - first_incident - last_incident @@ -3179,6 +3187,45 @@ components: $ref: '#/components/schemas/IssueBuildItem' title: IssueBuildsResponse type: array + IssueCheckout: + properties: + start_time: + $ref: '#/components/schemas/Timestamp' + git_commit_hash: + anyOf: + - $ref: '#/components/schemas/Checkout__GitCommitHash' + - type: 'null' + default: null + git_repository_url: + anyOf: + - $ref: '#/components/schemas/Checkout__GitRepositoryUrl' + - type: 'null' + default: null + git_repository_branch: + anyOf: + - $ref: '#/components/schemas/Checkout__GitRepositoryBranch' + - type: 'null' + default: null + git_commit_name: + anyOf: + - $ref: '#/components/schemas/Checkout__GitCommitName' + - type: 'null' + default: null + tree_name: + anyOf: + - $ref: '#/components/schemas/Checkout__TreeName' + - type: 'null' + default: null + checkout_id: + anyOf: + - type: string + - type: 'null' + default: null + title: Checkout Id + required: + - start_time + title: IssueCheckout + type: object IssueDetailsResponse: properties: field_timestamp: @@ -4528,6 +4575,22 @@ components: - issues title: TreeReportResponse type: object + TreeSeenData: + properties: + first_incident: + $ref: '#/components/schemas/Incident' + last_incident: + $ref: '#/components/schemas/Incident' + first_good_checkout: + anyOf: + - $ref: '#/components/schemas/IssueCheckout' + - type: 'null' + default: null + required: + - first_incident + - last_incident + title: TreeSeenData + type: object TreeSetItem: properties: tree_name: diff --git a/dashboard/src/components/IssueDetails/IncidentsSection.tsx b/dashboard/src/components/IssueDetails/IncidentsSection.tsx new file mode 100644 index 000000000..270c660f9 --- /dev/null +++ b/dashboard/src/components/IssueDetails/IncidentsSection.tsx @@ -0,0 +1,138 @@ +import type { JSX } from 'react'; +import { FormattedMessage } from 'react-intl'; + +import { ChevronRightAnimate } from '@/components/AnimatedIcons/Chevron'; +import LinkWithIcon from '@/components/LinkWithIcon/LinkWithIcon'; +import { TooltipDateTime } from '@/components/TooltipDateTime'; +import { TreeDetailsLink } from '@/components/TreeDetailsLink/TreeDetailsLink'; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from '@/components/ui/collapsible'; + +import { valueOrEmpty } from '@/lib/string'; + +import type { MessagesKey } from '@/locales/messages'; +import type { Checkout, Incident, TreeSeenData } from '@/types/issueExtras'; + +const CheckoutCell = ({ + title, + data, + time, + version, +}: { + title: MessagesKey; + data: Incident | Checkout; + time: Date; + version?: string; +}): JSX.Element => ( +
+
+ + + + + + + +
+ + {version !== undefined && ( + + )} +
+); + +const treeTitle = (tree: TreeSeenData): string => { + const name = valueOrEmpty(tree.last_incident.tree_name); + const branch = valueOrEmpty(tree.last_incident.git_repository_branch); + return `${name} / ${branch}`; +}; + +const TreeIncidentCollapsible = ({ + tree, +}: { + tree: TreeSeenData; +}): JSX.Element => { + const checkout = tree.first_good_checkout; + + return ( + + + + + {treeTitle(tree)} + + + +
+ + + {checkout ? ( + + ) : ( +
+ + + + +
+ )} +
+ + + ); +}; + +export const IncidentsSection = ({ + perTree, +}: { + perTree?: TreeSeenData[]; +}): JSX.Element | undefined => { + if (!perTree || perTree.length === 0) { + return; + } + + return ( +
+ + + + {perTree.map(tree => ( + + ))} +
+ ); +}; diff --git a/dashboard/src/components/IssueDetails/IssueDetails.tsx b/dashboard/src/components/IssueDetails/IssueDetails.tsx index db085ed19..de4748024 100644 --- a/dashboard/src/components/IssueDetails/IssueDetails.tsx +++ b/dashboard/src/components/IssueDetails/IssueDetails.tsx @@ -50,6 +50,8 @@ import { Badge } from '@/components/ui/badge'; import { MemoizedKcidevFooter } from '@/components/Footer/KcidevFooter'; +import { IncidentsSection } from './IncidentsSection'; + import { IssueDetailsTestSection } from './IssueDetailsTestSection'; import { IssueDetailsBuildSection } from './IssueDetailsBuildSection'; @@ -221,15 +223,6 @@ export const IssueDetails = ({ ]; }, [data, tagPills, issueCulprit]); - const sectionsData: ISection[] = useMemo(() => { - return [ - ...generalSections, - firstIncidentSection, - logspecSection, - miscSection, - ].filter(section => !!section); - }, [generalSections, logspecSection, miscSection, firstIncidentSection]); - const issueDetailsTabTitle = useMemo(() => { return formatMessage( { id: 'title.issueDetails' }, @@ -260,7 +253,15 @@ export const IssueDetails = ({
{breadcrumb} - + + + !!section)} + /> void; unformattedTitle?: string; titleIcon?: JSX.Element; + className?: string; } const LinkWithIcon = ({ @@ -25,6 +26,7 @@ const LinkWithIcon = ({ onClick, unformattedTitle, titleIcon, + className, }: ILinkWithIcon): JSX.Element => { const WrapperLink = link ? 'a' : 'div'; @@ -38,7 +40,9 @@ const LinkWithIcon = ({ }, [title, unformattedTitle]); return ( -
+
{(titleText || titleIcon) && (
{titleText && {titleText}} diff --git a/dashboard/src/locales/messages/index.ts b/dashboard/src/locales/messages/index.ts index e1e111107..3c584a889 100644 --- a/dashboard/src/locales/messages/index.ts +++ b/dashboard/src/locales/messages/index.ts @@ -258,9 +258,11 @@ export const messages = { 'issueDetails.culpritHarness': 'Harness', 'issueDetails.culpritTitle': 'Culprit', 'issueDetails.culpritTool': 'Tool', + 'issueDetails.firstGoodCheckout': 'First Good Checkout', 'issueDetails.firstIncidentData': 'First Incident Data', 'issueDetails.firstIncidentVersion': 'First Incident Version', 'issueDetails.id': 'Issue Id', + 'issueDetails.incidents': 'Incidents per Tree', 'issueDetails.issueDetails': 'Issue Details', 'issueDetails.issueListingInfo': 'The culprit for all issues listed is code', @@ -271,6 +273,7 @@ export const messages = { 'issueDetails.notFound': 'Issue not found', 'issueDetails.reportSubject': 'Report Subject', 'issueDetails.reportUrl': 'Report URL', + 'issueDetails.stillFailing': 'Still failing', 'issueDetails.version': 'Version', 'issueListing.culpritInfo': 'Layers of the execution stack responsible for the issue.\nIf all are false, the issue is considered invalid.', diff --git a/dashboard/src/types/issueExtras.ts b/dashboard/src/types/issueExtras.ts index 9ddb887ae..00cb06fd8 100644 --- a/dashboard/src/types/issueExtras.ts +++ b/dashboard/src/types/issueExtras.ts @@ -21,9 +21,26 @@ export type Incident = { issue_version?: string; }; +export type Checkout = { + start_time: Date; + git_commit_hash?: string; + git_repository_url?: string; + git_repository_branch?: string; + git_commit_name?: string; + tree_name?: string; + checkout_id?: string; +}; + +export type TreeSeenData = { + first_incident: Incident; + last_incident: Incident; + first_good_checkout?: Checkout; +}; + type TExtraIssuesData = { first_incident: Incident; last_incident: Incident; + per_tree?: TreeSeenData[]; versions: Record; };