ウェブスクレイピング (selenium - requests)

# ウェブスクレイピング
# python + selenimu + webDriver + BeautifulSoup
#
#python 3.8+
#
#conda install -c conda-forge selenium==4.15.1 
#   → https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/
#conda install -c anaconda beautifulsoup4==4.9.1

import sys
import time
import datetime
import traceback
from selenium import webdriver
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import requests as rq
import ssl, urllib3

##################################################
class CustomHttpAdapter (rq.adapters.HTTPAdapter):
  def __init__(self, ssl_context=None, **kwargs):
    self.ssl_context = ssl_context
    super().__init__(**kwargs)
 
  def init_poolmanager(self, connections, maxsize, block=False):
    self.poolmanager = urllib3.poolmanager.PoolManager(
         num_pools=connections
        ,maxsize=maxsize
        ,block=block
        ,ssl_context=self.ssl_context)

##################################################
def login_post():
  # open web browser
  print('open web browser')
  options = webdriver.EdgeOptions()
  options.add_argument("headless")
  options.add_argument('log-level=3') # INFO = 0, WARNING = 1, LOG_ERROR = 2, LOG_FATAL = 3.
  service = webdriver.EdgeService(executable_path='./msedgedriver.exe', service_args=['--log-level=SEVERE'])

  driver = webdriver.Edge(service=service, options = options)
  driver.set_window_size('1200', '1000')
  
  # login
  print ('login')
  driver.get('url')
  time.sleep(1)
  print (driver.current_url)
  driver.find_element(By.ID, 'user-name').send_keys('id')
  driver.find_element(By.ID, 'password').send_keys('pass')
  driver.find_element(By.ID, 'login').click()
  time.sleep(2)

  with rq.Session() as s:
    ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
    ctx.options |= 0x4
    s.mount('https://', CustomHttpAdapter(ctx))
    for cookies in driver.get_cookies(): # cookiejar
      s.cookies.set(cookies["name"], cookies["value"], **{"domain":cookies["domain"] ,"path":cookies["path"]})
    st = s.post('url',data=data)
    print (st.text)

  # close web browser
  print ('close web browser')
  driver.close()
  driver.quit()

Generic.List ソート

Dim ary As Generic.List(Of Integer)

ary.Sort() ' 正順(昇順)でソート default
ary.Sort(Function(x, y) y.CompareTo(x)) ' 逆順(降順)でソート

改行コード

UNIX / Linux / Mac 環境 → LF
Windows 環境 → CR+LF
(Mac OS X 以前の Mac → CR)

flask - jinja におけるテンプレート差込みの考え方

return render_template(page1.html) 



1. page1.html が呼出されると

2. extends 指定に基づいて base.html を読込み

3. base.html 内の block に同じ名前の block を差込む

4. その結果を返す

カンマ区切りの文字列作成

val sb = mutableListOf<String>()

sb.add("val1")

sb.add("val2")

sb.add("val3")

sb.add("val4")

sb.add("val5")

val sz = sb.joinToString(",")

Java8 以上

win32 C++ でWMI(Windows Management Instrumentation)を使用する

以下は指定IP(szIPaddr)に ping を打った時の status を取得する

// 主スレッドで呼出し
CoInitializeEx(NULL, COINIT_APARTMENTTHREADED);
CoInitializeSecurity(NULL, -1, NULL, NULL, RPC_C_AUTHN_LEVEL_DEFAULT, RPC_C_IMP_LEVEL_IMPERSONATE, NULL, EOAC_NONE, 0);

// Ping status
HRESULT hr;
CComPtr< IWbemLocator > pLocator;
hr = CoCreateInstance(CLSID_WbemAdministrativeLocator, NULL, CLSCTX_INPROC_SERVER, IID_IWbemLocator, (void**)(&pLocator));
if (SUCCEEDED(hr))
{
  CComPtr< IWbemServices > pService;
  //hr = pLocator->ConnectServer(L"root\\cimv2", NULL, NULL, NULL, WBEM_FLAG_CONNECT_USE_MAX_WAIT, NULL, NULL, &pService);
  hr = pLocator->ConnectServer(L"root\\cimv2", NULL, NULL, NULL, 0, NULL, NULL, &pService);
  if (SUCCEEDED(hr))
  {
    CString szSQL;
    szSQL.Format(_T("SELECT * FROM Win32_PingStatus WHERE Address='%s'"), szIPaddr);
    _bstr_t bzSQL(_bstr_t(CT2W((LPCTSTR)szSQL)));
    CComPtr< IEnumWbemClassObject > pEnumObj;
    hr = pService->ExecQuery(L"WQL", bzSQL, (WBEM_FLAG_RETURN_IMMEDIATELY|WBEM_FLAG_FORWARD_ONLY), NULL, &pEnumObj);
    if (SUCCEEDED(hr))
    {
      CComPtr< IWbemClassObject > pObj;
      ULONG uReturned;
      //hr = pEnumObj->Reset();
      hr = pEnumObj->Next(WBEM_INFINITE, 1, &pObj, &uReturned);
      if (!FAILED(hr))
      {
        _variant_t vtBuf;
        hr = pObj->Get(L"StatusCode", 0, &vtBuf, 0, 0);
        pObj.Release();
      }
      pEnumObj.Release();
    }
    pService.Release();
  }
  pLocator.Release();
}

VB.net での nothing (null) の扱い

勝手に初期値に変換される!

Dim dt as Date = nothing
としても dt は初期値がセットされている (nothingではない)
if (dt is Nothing) then
 → エラー

Nothing を許容するには
Dim dt? as Date = nothing  
変数名のおしりに"?" をつけると nothing が代入できる
if (dt is Nothing) then
 → OK


Date型またはNothingを返す関数
Function GetDate() as Nullable(of Date)


Databaseへの登録
Nothing と DBNull.value は異なる

adoCmd.parameter.Add("DATE_VALUE",type.date).value = IIf (dt is nothing, DBNull.value, dt)

-------------------------------------------------------------

Sub Test() 
    Dim dt1 As Date = Nothing
    Dim dt2? As Date = Nothing

    'If (dt1 Is Nothing) Then
    '    Console.WriteLine("dt1 is nothing")
    'End If
    If (dt2 Is Nothing) Then
        Console.WriteLine("dt2 is nothing")
    End If

    Console.WriteLine(String.Format("value1: {0}", dt1.ToString()))
    Console.WriteLine(String.Format("value2: {0}", dt2.ToString()))

    Console.WriteLine(String.Format("F1: {0}", F1().ToString()))
    Console.WriteLine(String.Format("F2: {0}", F2().ToString()))
End Sub

Function F1() As Date
    Return Nothing
End Function

Function F2() As Nullable(Of Date)
    Return Nothing
End Function