服务器记录(249)

最近优化了些服务器做下记录

1. httpclient的使用

   httpclient不能立即关闭，会占用资源，并发量一大就耗尽。所以.net core 会使用如下:

public class RestClient : IDisposable
   {
       private readonly ILogger Logger = LoggerManager.GetLogger("ApiClient");

       private HttpClient _httpClient = null;
       const string TASKAPI20 = "api/v2/task";
       const string TASKAPI30 = "api/v3/task";
       const string MATRIXAPI20 = "api/v2/matrix";
       const string USERAPI20 = "api/v2/user";
       const string DEVICEAPI30 = "api/v3/device";
       const string DEVICEAPI20 = "api/v2/device";

       private string IngestDbUrl { get; set; }
       private string CmServerUrl { get; set; }

       private bool _disposed;
       public RestClient(HttpClient httpClient, string ingesturl, string cmurl)
       {
           _disposed = false;
           _httpClient = httpClient != null? httpClient : new HttpClient();
           _httpClient.DefaultRequestHeaders.Connection.Clear();
           _httpClient.DefaultRequestHeaders.ConnectionClose = false;
           _httpClient.Timeout = TimeSpan.FromSeconds(15);
           _httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
           _httpClient.DefaultRequestHeaders.Add("sobeyhive-http-system", "INGESTSERVER");
           _httpClient.DefaultRequestHeaders.Add("sobeyhive-http-site", "S1");
           _httpClient.DefaultRequestHeaders.Add("sobeyhive-http-tool", "INGESTSERVER");
           IngestDbUrl = ingesturl;
           CmServerUrl = cmurl;
           
       }
       public void Dispose()
       {
           Dispose(true);
           GC.SuppressFinalize(this);
       }

       ~RestClient()
       {
           //必须为false
           Dispose(false);
       }

       protected virtual void Dispose(bool disposing)
       {
           if (_disposed) return;
           if (disposing)
           {

           }
           if (_httpClient != null)
           {
               _httpClient.Dispose();
               _httpClient = null;
           }
           _disposed = true;
       }

       public Dictionary<string, string> GetTokenHeader(string usertoken)
       {
           return new Dictionary<string, string>() { 
               {"sobeyhive-http-token", usertoken }
           };
       }

       public Dictionary<string, string> GetCodeHeader(string usertoken)
       {
           return new Dictionary<string, string>() {
               {"sobeyhive-http-secret", RSAHelper.RSAstr()},
               {"current-user-code", usertoken }
           };
       }
       public Dictionary<string, string> GetIngestHeader()
       {
           return new Dictionary<string, string>() {
               {"sobeyhive-ingest-signature", Base64SQL.ToBase64String($"ingest_server;{DateTime.Now}")},
           };
       }

       public async Task<TResponse> PostAsync<TResponse>(string url, object body, string method = "POST", NameValueCollection queryString = null, int timeout = 60)
           where TResponse : class, new()
       {
           TResponse response = null;
           try
           {
               string json = JsonHelper.ToJson(body);
               HttpClient client = _httpClient;
               if (queryString == null)
               {
                   queryString = new NameValueCollection();
               }
               if (String.IsNullOrEmpty(method))
               {
                   method = "POST";
               }
               url = CreateUrl(url, queryString);
               //Logger.Debug("请求：{0} {1}", method, url);
               byte[] strData = Encoding.UTF8.GetBytes(json);
               MemoryStream ms = new MemoryStream(strData);
               using (StreamContent sc = new StreamContent(ms))
               {
                   sc.Headers.ContentType = new MediaTypeWithQualityHeaderValue("application/json");

                   //foreach (var item in _httpClient.DefaultRequestHeaders)
                   //{
                   //    Logger.Error("header :  " + item.Key + ":" + item.Value.FirstOrDefault());
                   //    foreach(var test in item.Value)
                   //    {
                   //        Logger.Error("test :  " + test);
                   //    }
                   //}

                   var res = await client.PostAsync(url, sc).ConfigureAwait(true);
                   byte[] rData = await res.Content.ReadAsByteArrayAsync().ConfigureAwait(true);
                   string rJson = Encoding.UTF8.GetString(rData);
                   Logger.Info("url body response：\r\n{0} {1} {2}", url, json, rJson);
                   response = JsonHelper.ToObject<TResponse>(rJson);
                   return response;
               }
                  
           }
           catch (System.Exception e)
           {
               TResponse r = new TResponse();
               Logger.Error("请求异常：\r\n{0} {1}", e.ToString(), url);
               throw;
           }
       }

       public async Task<string> PostAsync(string url, string body, string method = "POST", NameValueCollection queryString = null, int timeout = 60)
       {
           string response = null;
           try
           {
               string json = body;
               HttpClient client = _httpClient;
               if (queryString == null)
               {
                   queryString = new NameValueCollection();
               }
               if (String.IsNullOrEmpty(method))
               {
                   method = "POST";
               }
               url = CreateUrl(url, queryString);
               //Logger.Debug("请求：{0} {1}", method, url);
               byte[] strData = Encoding.UTF8.GetBytes(json);
               MemoryStream ms = new MemoryStream(strData);
               using (StreamContent sc = new StreamContent(ms))
               {
                   sc.Headers.ContentType = new MediaTypeWithQualityHeaderValue("application/json");
                   var res = await client.PostAsync(url, sc).ConfigureAwait(true);
                   if (res.Content == null || res.Content.Headers.ContentLength == 0)
                   {
                       response = "";
                   }
                   else
                   {
                       byte[] rData = await res.Content.ReadAsByteArrayAsync().ConfigureAwait(true);
                       string rJson = Encoding.UTF8.GetString(rData);
                       //Logger.Debug("应答：\r\n{0}", rJson);
                       response = rJson;
                   }
               }
                   
           }
           catch (System.Exception e)
           {
               response = "ERROR";
               Logger.Error("请求异常：\r\n{0} {1}", e.ToString(), url);
           }
           return response;
       }

       public async Task<TResponse> PutAsync<TResponse>(string url, object body, Dictionary<string, string> header, NameValueCollection queryString = null)
       {
           TResponse response = default(TResponse);
           try
           {
               string json = JsonHelper.ToJson(body);
               HttpClient client = _httpClient;
               if (queryString == null)
               {
                   queryString = new NameValueCollection();
               }

               url = CreateUrl(url, queryString);
               //Logger.Debug("请求：{0} {1}", method, url);
               byte[] strData = Encoding.UTF8.GetBytes(json);
               MemoryStream ms = new MemoryStream(strData);
               using (StreamContent sc = new StreamContent(ms))
               {
                   sc.Headers.ContentType = new MediaTypeWithQualityHeaderValue("application/json");
                   if (header != null)
                   {
                       foreach (var item in header)
                       {
                           sc.Headers.Add(item.Key, item.Value);
                       }
                   }

                   var res = await client.PutAsync(url, sc).ConfigureAwait(true);
                   byte[] rData = await res.Content.ReadAsByteArrayAsync().ConfigureAwait(true);
                   string rJson = Encoding.UTF8.GetString(rData);
                   //Logger.Debug("应答：\r\n{0}", rJson);
                   response = JsonHelper.ToObject<TResponse>(rJson);
                   return response;
               }

           }
           catch (System.Exception e)
           {
               Logger.Error("请求异常：\r\n{0} {1}", e.ToString(), url);
               throw;
           }
       }

       public async Task<TResponse> DeleteAsync<TResponse>(string url, Dictionary<string, string> header, NameValueCollection queryString = null)
           where TResponse : class, new()
       {
           TResponse response = default(TResponse);
           try
           {
               HttpClient client = _httpClient;
               if (queryString == null)
               {
                   queryString = new NameValueCollection();
               }

               url = CreateUrl(url, queryString);
               //Logger.Debug("请求：{0} {1}", method, url);

               using (var requestMessage = new HttpRequestMessage(HttpMethod.Delete, url))
               {
                   if (header != null)
                   {
                       foreach (var item in header)
                       {
                           requestMessage.Headers.Add(item.Key, item.Value);
                       }
                   }
                   var backinfo = await client.SendAsync(requestMessage).ConfigureAwait(true);
                   var rJson = await backinfo.Content.ReadAsStringAsync().ConfigureAwait(true);
                   Logger.Info("url response：\r\n{0} {1}", url, rJson);
                   response = JsonHelper.ToObject<TResponse>(rJson);
               }

           }
           catch (System.Exception e)
           {
               TResponse r = new TResponse();
               Logger.Error("请求异常：\r\n{0}", e.ToString(), url);
               return r;
           }
           return response;
       }

       public async Task<TResponse> GetAsync<TResponse>(string url, NameValueCollection queryString, Dictionary<string, string> header)
                   where TResponse : class, new()
       {
           TResponse response = null;
           try
           {
               HttpClient client = _httpClient;
               if (queryString != null)
               {
                   url = CreateUrl(url, queryString);
               }
               
               //Logger.Debug("请求：{0} {1}", "GET", url);
               using (var requestMessage = new HttpRequestMessage(HttpMethod.Get, url))
               {
                   if (header != null)
                   {
                       foreach (var item in header)
                       {
                           requestMessage.Headers.Add(item.Key, item.Value);
                       }
                   }
                   var backinfo = await client.SendAsync(requestMessage).ConfigureAwait(true);
                   var rJson = await backinfo.Content.ReadAsStringAsync().ConfigureAwait(true);
                   Logger.Info("url response：\r\n{0} {1}", url, rJson);
                   response = JsonHelper.ToObject<TResponse>(rJson);
               }
               
           }
           catch (System.Exception e)
           {
               TResponse r = new TResponse();
               Logger.Error("请求异常：\r\n{0} {1}", e.ToString(), url);
               return r;
           }
           return response;
       }

       //public async Task<TResponse> GetAsync<TResponse>(string url, NameValueCollection queryString)
       //            where TResponse : class, new()
       //{
       //    TResponse response = null;
       //    try
       //    {
       //        HttpClient client = _httpClient;
       //        if (queryString != null)
       //        {
       //            url = CreateUrl(url, queryString);
       //        }
       //        
       //        //Logger.Debug("请求：{0} {1}", "GET", url);
       //        byte[] rData = await client.GetByteArrayAsync(url).ConfigureAwait(true);
       //        string rJson = Encoding.UTF8.GetString(rData);
       //        Logger.Info("url response：\r\n{0} {1}", url, rJson);
       //        response = JsonHelper.ToObject<TResponse>(rJson);
       //    }
       //    catch (System.Exception )
       //    {
       //        TResponse r = new TResponse();
       //        //Logger.Error("请求异常：\r\n{0}", e.ToString());
       //        return r;
       //    }
       //    return response;
       //}
       public async Task<TResponse> PostAsync<TResponse>(string url, object body, Dictionary<string, string> header, string method = null, NameValueCollection queryString = null)
       {
           TResponse response = default(TResponse);
           try
           {
               string json = JsonHelper.ToJson(body);
               HttpClient client = _httpClient;
               if (queryString == null)
               {
                   queryString = new NameValueCollection();
               }

               url = CreateUrl(url, queryString);
               if (String.IsNullOrEmpty(method))
               {
                   method = "POST";
               }
               //Logger.Debug("请求：{0} {1}", method, url);
               byte[] strData = Encoding.UTF8.GetBytes(json);
               MemoryStream ms = new MemoryStream(strData);
               using (StreamContent sc = new StreamContent(ms))
               {
                   sc.Headers.ContentType = new MediaTypeWithQualityHeaderValue("application/json");
                   if (header != null)
                   {
                       foreach (var item in header)
                       {
                           sc.Headers.Add(item.Key, item.Value);
                       }
                   }
                   
                   var res = await client.PostAsync(url, sc).ConfigureAwait(true);
                   byte[] rData = await res.Content.ReadAsByteArrayAsync().ConfigureAwait(true);
                   string rJson = Encoding.UTF8.GetString(rData);
                   //Logger.Debug("应答：\r\n{0}", rJson);
                   response = JsonHelper.ToObject<TResponse>(rJson);
                   return response;
               }
                   
           }
           catch (System.Exception )
           {
               //Logger.Error("请求异常：\r\n{0}", e.ToString());
               throw;
           }
       }

       //public async Task<string> PostAsync(string url, object body, string method, NameValueCollection queryString)
       //{
       //    string response = null;
       //    try
       //    {
       //        string json = JsonHelper.ToJson(body);
       //        HttpClient client = _httpClient;
       //        if (queryString == null)
       //        {
       //            queryString = new NameValueCollection();
       //        }

       //        url = CreateUrl(url, queryString);
       //        if (String.IsNullOrEmpty(method))
       //        {
       //            method = "POST";
       //        }
       //        //Logger.Debug("请求：{0} {1}", method, url);
       //        byte[] strData = Encoding.UTF8.GetBytes(json);
       //        MemoryStream ms = new MemoryStream(strData);
       //        using (StreamContent sc = new StreamContent(ms))
       //        {
       //            sc.Headers.ContentType = new MediaTypeWithQualityHeaderValue("application/json");
       //            var res = await client.PostAsync(url, sc).ConfigureAwait(true);
       //            byte[] rData = await res.Content.ReadAsByteArrayAsync().ConfigureAwait(true);
       //            string rJson = Encoding.UTF8.GetString(rData);
       //            //Logger.Debug("应答：\r\n{0}", rJson);
       //            response = rJson;
       //            return response;
       //        }
                   
       //    }
       //    catch (System.Exception )
       //    {
       //        //Logger.Error("请求异常：\r\n{0}", e.ToString());
       //        throw;
       //    }
       //}

       public async Task<TResult> PostWithTokenAsync<TResult>(string url, object body, string token, string userId = null, string method = "Post")
       {
           //Stopwatch sw = new Stopwatch();
           //sw.Start();

           string apiUrl = $"{url}";
           HttpMethod hm = new HttpMethod(method);

           using (var request = new HttpRequestMessage(hm, apiUrl))
           {
               if (!String.IsNullOrEmpty(token))
               {
                   request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);

               }
               if (!String.IsNullOrEmpty(userId))
               {
                   request.Headers.Add("User", userId);
               }

               string json = "";
               if (body != null)
               {
                   json = Newtonsoft.Json.JsonConvert.SerializeObject(body);
               }
               request.Content = new StringContent(json);
               request.Content.Headers.ContentType = new MediaTypeWithQualityHeaderValue("application/json");

               var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead).ConfigureAwait(true);
               if (response.StatusCode == System.Net.HttpStatusCode.Unauthorized ||
                   response.StatusCode == System.Net.HttpStatusCode.Forbidden)
               {
                   throw new UnauthorizedAccessException("验证失败");
               }
               try
               {
                   response.EnsureSuccessStatusCode();
                   string str = await response.Content.ReadAsStringAsync().ConfigureAwait(true);

                   //sw.Stop();
                   //if (sw.ElapsedMilliseconds >= 1000)
                   //{
                   //slowLogger.Warn("请求时间超过一秒：POST {0} {1}", apiUrl, sw.ElapsedMilliseconds);
                   //}

                   return Newtonsoft.Json.JsonConvert.DeserializeObject<TResult>(str);
               }
               catch (Exception )
               {
                   //logger.Error("Post 失败：{0}\r\n{1}", url, e.ToString());
                   string str = await response.Content.ReadAsStringAsync().ConfigureAwait(true);
                   //logger.Error(str);
                   throw;
               }
           }
               
       }

       public async Task<TResult> SubmitFormAsync<TResult>(string url, Dictionary<string, string> formData, string method = "Post")
       {
           HttpMethod hm = new HttpMethod(method);
           using (var request = new HttpRequestMessage(hm, url))
           {
               request.Content = new FormUrlEncodedContent(formData);
               var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead).ConfigureAwait(true);
               if (response.StatusCode == System.Net.HttpStatusCode.Unauthorized ||
                   response.StatusCode == System.Net.HttpStatusCode.Forbidden)
               {
                   throw new UnauthorizedAccessException("验证失败");
               }
               response.EnsureSuccessStatusCode();
               string str = await response.Content.ReadAsStringAsync().ConfigureAwait(true);
               return Newtonsoft.Json.JsonConvert.DeserializeObject<TResult>(str);
           }
               
       }


       public static string CreateUrl(string url, NameValueCollection qs)
       {
           if (qs != null && qs.Count > 0)
           {
               StringBuilder sb = new StringBuilder();
               List<string> kl = qs.AllKeys.ToList();
               foreach (string k in kl)
               {
                   if (sb.Length > 0)
                   {
                       sb.Append("&");
                   }
                   sb.Append(k).Append("=");
                   if (!String.IsNullOrEmpty(qs[k]))
                   {

                       sb.Append(System.Net.WebUtility.UrlEncode(qs[k]));
                   }
               }

               if (url != null)
               {
                   if (url.Contains("?"))
                   {
                       url = url + "&" + sb.ToString();
                   }
                   else
                   {
                       url = url + "?" + sb.ToString();
                   }
               }
               
           }

           return url;

       }

       #region Global
       public async Task<List<UserLoginInfo>> GetAllUserLoginInfosAsync()
       {
           var back = await AutoRetry.RunAsync<ResponseMessage<List<UserLoginInfo>>>(() =>
           {
               return GetAsync<ResponseMessage<List<UserLoginInfo>>>(
                   $"{IngestDbUrl}/{USERAPI20}/userlogininfo/all", null, GetIngestHeader()
                   );
           }).ConfigureAwait(true);

           if (back != null)
           {
               return back.Ext;
           }
           return null;
       }
       #endregion

       #region Task

       public async Task<TaskSource> GetTaskSourceByTaskIdAsync(int taskid)
       {
           var back = await AutoRetry.RunAsync<ResponseMessage<TaskSource>>(() =>
           {
               return GetAsync<ResponseMessage<TaskSource>>(
                   $"{IngestDbUrl}/{TASKAPI20}/tasksource/{taskid}",
                   null, GetIngestHeader());
           }).ConfigureAwait(true);

           if (back != null)
           {
               return back.Ext;
           }
           return TaskSource.emUnknowTask;
       }

       public async Task<DispatchTask> GetTaskDBAsync(int taskid)
       {
           var back = await GetAsync<ResponseMessage<DispatchTask>>(
                   $"{IngestDbUrl}/{TASKAPI30}/db/{taskid}", null, GetIngestHeader()
                   ).ConfigureAwait(true);

           if (back != null)
           {
               return back.Ext;
           }
           return null;
       }

       public async Task<TaskContent> ReScheduleTaskChannelAsync(int oldtaskid)
       {
           var back = await AutoRetry.RunAsync<ResponseMessage<TaskContent>>(() =>
           {

               return PutAsync<ResponseMessage<TaskContent>>(
                   $"{IngestDbUrl}/{TASKAPI30}/reschedule/channel/{oldtaskid}", null,
                   GetIngestHeader());
           }).ConfigureAwait(true);

           if (back != null && back.IsSuccess())
           {
               return back.Ext;
           }
           return null;
       }
       #endregion
     
   }

1. 在异步方法中，不要使用 Thread.Sleep；在同步方法中，不要使用Task.Delay ，否则可能出现线程死锁，结果难出来。

2. 吞吐量（TPS）、QPS（每秒查询率）、并发数、响应时间（RT）
   当时为了增加qps，把所有webapi接口都改成异步请求