Observability of your Cloud Software using Azure Application Insights

devNetNoord 2023

Transcript

1. APPLICATION INSIGHTS GUIDE, OVERVIEW, TIPS AND TRICKS by Mart de Graaf 1
3. BASIC AZURE APPLICATION INSIGHTS 1. requests 2. traces 3. exceptions 4. dependencies 5. pageViews 3
4. AZURE LOG ANALYTICS WORKSPACE AI LAW requests AppRequests traces AppTraces exceptions AppExceptions dependencies AppDependencies pageViews AppPageViews 4
5. KUSTO IN MICROSOFT AZURE 1. Azure Application Insights and Log Analytics 2. Azure Monitor 3. Azure Resource Graph Explorer 5
6. AZURE DATA EXPLORER KUSTO QUERY LANGUAGE (KQL) 1. 2. https://learn.microsoft.com/en-us/azure/data- explorer/kusto/query/ https://learn.microsoft.com/en-us/azure/data- explorer/kusto/query/sqlcheatsheet? source=recommendations 6
7. RENDERING KQL 7
8. LINECHART exceptions | summarize Count() | render linechart 1 2 3 8
9. STACKEDAREACHART exceptions | summarize Count() | render stackedareachart 1 2 3 9
10. SCATTERCHART demo_series2 | extend series_fit_2lines(y), series_fit_line(y) | render scatterchart with(xcolumn=x) 1 2 3 10
11. WRITE LOGS try { await _client.Delete(..); _logger.LogInformation("Delete completed `{sequenceNum } public async Task Delete(long sequenceNumber) 1 { 2 _logger.LogInformation("Deleting `{sequenceNumber}`.", s 3 4 5 6 7 8 catch (InvalidOperationException ex) 9 when (ex.Message.Equals($"The scheduled message with S 10 { 11 _logger.LogError(ex, "Already cancelled {sequenceNumbe 12 } 13 } 14 11
12. WRITE LOGS try { await _client.Delete(..); _logger.LogInformation("Delete completed `{sequenceNum } public async Task Delete(long sequenceNumber) 1 { 2 _logger.LogInformation("Deleting `{sequenceNumber}`.", s 3 4 5 6 7 8 catch (InvalidOperationException ex) 9 when (ex.Message.Equals($"The scheduled message with S 10 { 11 _logger.LogError(ex, "Already cancelled {sequenceNumbe 12 } 13 } 14 catch (InvalidOperationException ex) when (ex.Message.Equals($"The scheduled message with S { _logger.LogError(ex, "Already cancelled {sequenceNumbe } public async Task Delete(long sequenceNumber) 1 { 2 _logger.LogInformation("Deleting `{sequenceNumber}`.", s 3 try 4 { 5 await _client.Delete(..); 6 _logger.LogInformation("Delete completed `{sequenceNum 7 } 8 9 10 11 12 13 } 14 11.1
13. WRITE LOGS try { await _client.Delete(..); _logger.LogInformation("Delete completed `{sequenceNum } public async Task Delete(long sequenceNumber) 1 { 2 _logger.LogInformation("Deleting `{sequenceNumber}`.", s 3 4 5 6 7 8 catch (InvalidOperationException ex) 9 when (ex.Message.Equals($"The scheduled message with S 10 { 11 _logger.LogError(ex, "Already cancelled {sequenceNumbe 12 } 13 } 14 catch (InvalidOperationException ex) when (ex.Message.Equals($"The scheduled message with S { _logger.LogError(ex, "Already cancelled {sequenceNumbe } public async Task Delete(long sequenceNumber) 1 { 2 _logger.LogInformation("Deleting `{sequenceNumber}`.", s 3 try 4 { 5 await _client.Delete(..); 6 _logger.LogInformation("Delete completed `{sequenceNum 7 } 8 9 10 11 12 13 } 14 when (ex.Message.Equals($"The scheduled message with S _logger.LogError(ex, "Already cancelled {sequenceNumbe public async Task Delete(long sequenceNumber) 1 { 2 _logger.LogInformation("Deleting `{sequenceNumber}`.", s 3 try 4 { 5 await _client.Delete(..); 6 _logger.LogInformation("Delete completed `{sequenceNum 7 } 8 catch (InvalidOperationException ex) 9 10 { 11 12 } 13 } 14 11.2
14. UNIT TEST YOUR LOGGING var logger = A.Fake<ILogger<SystemUnderTest>>(); var sut = new SystemUnderTest(logger); //Arrange 1 2 3 4 //Act 5 await sut.Delete(1); 6 7 //Assert 8 logger.VerifyLogged(LogLevel.Information, "Deleting 1"); 9 logger.VerifyLogged(LogLevel.Error, "Already cancelled 1") 10 12
15. UNIT TEST YOUR LOGGING var logger = A.Fake<ILogger<SystemUnderTest>>(); var sut = new SystemUnderTest(logger); //Arrange 1 2 3 4 //Act 5 await sut.Delete(1); 6 7 //Assert 8 logger.VerifyLogged(LogLevel.Information, "Deleting 1"); 9 logger.VerifyLogged(LogLevel.Error, "Already cancelled 1") 10 await sut.Delete(1); //Arrange 1 var logger = A.Fake<ILogger<SystemUnderTest>>(); 2 var sut = new SystemUnderTest(logger); 3 4 //Act 5 6 7 //Assert 8 logger.VerifyLogged(LogLevel.Information, "Deleting 1"); 9 logger.VerifyLogged(LogLevel.Error, "Already cancelled 1") 10 12.1
16. UNIT TEST YOUR LOGGING var logger = A.Fake<ILogger<SystemUnderTest>>(); var sut = new SystemUnderTest(logger); //Arrange 1 2 3 4 //Act 5 await sut.Delete(1); 6 7 //Assert 8 logger.VerifyLogged(LogLevel.Information, "Deleting 1"); 9 logger.VerifyLogged(LogLevel.Error, "Already cancelled 1") 10 await sut.Delete(1); //Arrange 1 var logger = A.Fake<ILogger<SystemUnderTest>>(); 2 var sut = new SystemUnderTest(logger); 3 4 //Act 5 6 7 //Assert 8 logger.VerifyLogged(LogLevel.Information, "Deleting 1"); 9 logger.VerifyLogged(LogLevel.Error, "Already cancelled 1") 10 logger.VerifyLogged(LogLevel.Information, "Deleting 1"); logger.VerifyLogged(LogLevel.Error, "Already cancelled 1") //Arrange 1 var logger = A.Fake<ILogger<SystemUnderTest>>(); 2 var sut = new SystemUnderTest(logger); 3 4 //Act 5 await sut.Delete(1); 6 7 //Assert 8 9 10 12.2
17. DEMO APPLICATION 13
18. GitHub: Azure Portal martdegraaf/kql-demo 14
19. SAMPLING 15
20. HOST.JSON "logLevel": { // Default settings "default": "Warning", // For all functions "Host.Results": "Information", "Host.Aggregator": "Information", "Function": "Information", "Microsoft": "Warning" }, { 1 "version": "2.0", 2 "logging": { 3 4 5 6 7 8 9 10 11 12 "applicationInsights": { 13 "EnableDependencyTracking": true, 14 "DependencyTrackingOptions": { 15 " bl S lC dT tI t t ti " t 16 16
21. HOST.JSON "logLevel": { // Default settings "default": "Warning", // For all functions "Host.Results": "Information", "Host.Aggregator": "Information", "Function": "Information", "Microsoft": "Warning" }, { 1 "version": "2.0", 2 "logging": { 3 4 5 6 7 8 9 10 11 12 "applicationInsights": { 13 "EnableDependencyTracking": true, 14 "DependencyTrackingOptions": { 15 " bl S lC dT tI t t ti " t 16 "applicationInsights": { "EnableDependencyTracking": true, "DependencyTrackingOptions": { "enableSqlCommandTextInstrumentation": true }, "samplingSettings": { "isEnabled": true, "excludedTypes": "Request, Exception" } } } Host.Aggregator : Information , 9 "Function": "Information", 10 "Microsoft": "Warning" 11 }, 12 13 14 15 16 17 18 19 20 21 22 23 } 24 16.1
22. DISABLE SAMPLING AzureFunctionsJobHost__logging__applicationInsights__sampli 1 17
23. BUSINESS CASES 18
24. FIND BROKEN QUERIES dependencies | where type == "SQL" | where data contains "1 = 0" | summarize Occurances=count(), OpertationIds=make_list(op | project Occurances, OpertationIds, data // gives every SQL-query that contains 1 = 0, which is ins // Occurances Times a query slowed in given // OperationIds List of example operationIds t // data The actual query executed //Example what type of Dapper query this would happen? 1 2 3 4 5 6 7 8 9 10 11 12 13 19
25. COUNT TRACES PER HOUR traces | where message has "Executed MartTimerFunction" or message has "Executed MartHttpFunction" or message has "Executed MartQueueFunction" | summarize count() by Date = bin(timestamp, 1h) | render columnchart // Extra info that could be added // Operation_Id, AppRoleName, Date=bin(TimeGenerated, 1h),s 1 2 3 4 5 6 7 8 9 20
26. EXCEPTIONS DISTINCT BY OUTERMESSAGE exceptions | extend reservation = tostring(customDimensions.RequestPat | distinct reservation, outerMessage | project reservation, outerMessage // tostring is important on customDimensions // I wanted to know what endpoints GET /{id} where called w // Then i communicated those Ids to the person who could fi 1 2 3 4 5 6 7 8 9 21
27. EXCEPTIONS BY OCCURRENCE WITH MULTIPLE AI-RESOURCES ( AppExceptions | parse _ResourceId with * '/components/' ServiceName | project OperationId, AppRoleName, ProblemId, OuterMessage, ServiceName, _SubscriptionId, ClientType, OuterMethod ) 1 2 3 4 5 6 7 8 9 10 11 12 13 | where ServiceName in ( 14 "mart-ai" 15 ) 16 22
28. EXCEPTIONS BY OCCURRENCE WITH MULTIPLE AI-RESOURCES ( AppExceptions | parse _ResourceId with * '/components/' ServiceName | project OperationId, AppRoleName, ProblemId, OuterMessage, ServiceName, _SubscriptionId, ClientType, OuterMethod ) 1 2 3 4 5 6 7 8 9 10 11 12 13 | where ServiceName in ( 14 "mart-ai" 15 ) 16 | where ServiceName in ( "mart-ai" ) ProblemId, 7 OuterMessage, 8 ServiceName, 9 _SubscriptionId, 10 ClientType, 11 OuterMethod 12 ) 13 14 15 16 | summarize 17 AnyOuterMessage = take_any(OuterMessage), 18 AnyOuterMethod = take_any(OuterMethod), 19 OperationIds= make_list(OperationId, 5), 20 uniqueOperations=dcount(OperationId) 21 by ServiceName, AppRoleName, ProblemId 22 | order by uniqueOperations desc ServiceName AppRoleName 23 22.1
29. EXCEPTIONS BY OCCURRENCE WITH MULTIPLE AI-RESOURCES ( AppExceptions | parse _ResourceId with * '/components/' ServiceName | project OperationId, AppRoleName, ProblemId, OuterMessage, ServiceName, _SubscriptionId, ClientType, OuterMethod ) 1 2 3 4 5 6 7 8 9 10 11 12 13 | where ServiceName in ( 14 "mart-ai" 15 ) 16 | where ServiceName in ( "mart-ai" ) ( 1 AppExceptions 2 | parse _ResourceId with * '/components/' ServiceName 3 | project 4 OperationId, 5 AppRoleName, 6 ProblemId, 7 OuterMessage, 8 ServiceName, 9 _SubscriptionId, 10 ClientType, 11 OuterMethod 12 ) 13 14 15 16 | summarize AnyOuterMessage = take_any(OuterMessage), AnyOuterMethod = take_any(OuterMethod), OperationIds= make_list(OperationId, 5), uniqueOperations=dcount(OperationId) by ServiceName, AppRoleName, ProblemId OuterMethod 12 ) 13 | where ServiceName in ( 14 "mart-ai" 15 ) 16 17 18 19 20 21 22 | order by uniqueOperations desc, ServiceName, AppRoleName 23 | project 24 uniqueOperations, 25 ServiceName, 26 AppRoleName, 27 22.2
30. EXCEPTIONS BY OCCURRENCE WITH MULTIPLE AI-RESOURCES ( AppExceptions | parse _ResourceId with * '/components/' ServiceName | project OperationId, AppRoleName, ProblemId, OuterMessage, ServiceName, _SubscriptionId, ClientType, OuterMethod ) 1 2 3 4 5 6 7 8 9 10 11 12 13 | where ServiceName in ( 14 "mart-ai" 15 ) 16 | where ServiceName in ( "mart-ai" ) ( 1 AppExceptions 2 | parse _ResourceId with * '/components/' ServiceName 3 | project 4 OperationId, 5 AppRoleName, 6 ProblemId, 7 OuterMessage, 8 ServiceName, 9 _SubscriptionId, 10 ClientType, 11 OuterMethod 12 ) 13 14 15 16 ( 1 AppExceptions 2 | parse _ResourceId with * '/components/' ServiceName 3 | project 4 OperationId, 5 AppRoleName, 6 ProblemId, 7 OuterMessage, 8 ServiceName, 9 _SubscriptionId, 10 ClientType, 11 OuterMethod 12 ) 13 | where ServiceName in ( 14 "mart-ai" 15 ) 16 | order by uniqueOperations desc, ServiceName, AppRoleName | project uniqueOperations, ServiceName, AppRoleName, AnyOuterMessage, AnyOuterMethod, ProblemId, OperationIds ) 16 | summarize 17 AnyOuterMessage = take_any(OuterMessage), 18 AnyOuterMethod = take_any(OuterMethod), 19 OperationIds= make_list(OperationId, 5), 20 uniqueOperations=dcount(OperationId) 21 by ServiceName, AppRoleName, ProblemId 22 23 24 25 26 27 28 29 30 31 22.3
31. PERFORMANCE BY CALLNAME requests | where cloud_RoleName == "p-wapp-mart" | where duration > 5000 | summarize amount=dcount(operation_Id), operationIds = mak | order by name | render columnchart with (title = "Performance van MartApp 1 2 3 4 5 6 7 23
32. SLOW RUNNING SQL QUERIES dependencies | where type == "SQL" | where duration > 25000 | summarize Occurances=count(), OperationIds=make_list(ope | project Occurances, MaxPerformanceBucket, AvgDuration, O // gives every SQL-query that took longer than 25s // Occurances Times a query slowed in given // MaxPerformanceBucket ex: 30sec-1min // AvgDuration ex: 29,753.374 time in ms // OperationIds List of example operationIds t // data The actual query executed 1 2 3 4 5 6 7 8 9 10 11 12 24
33. SHOW CHAIN RESPONSE TIMES traces | where operation_Name == "BookingPlaced" or message has " | extend reservationNumber = iff(isnull(customDimensions.p | where not(isnull(reservationNumber)) | summarize Min = min(timestamp), Max = max(timestamp), Du | order by Duration asc | extend ResponseTime = tolong(Duration) / 10000000 | extend ResponseTimeMinutes = ResponseTime / 60 | render scatterchart // renders scatterchart with dots and information. // Important is the tostring() op customDimensions en ande 1 2 3 4 5 6 7 8 9 10 11 12 13 25
34. AZURE RESOURCES WITH TAGS resources | where isnotempty(tags) | extend teamTag = tostring(tags["team"]) | extend serviceTag = tostring(tags["Service"]) | extend serviceTag2 = tostring(tags["service"]) | where name like "prefix-p-" | project name, teamTag, serviceTag | order by ['serviceTag'] desc 1 2 3 4 5 6 7 8 26